diff --git "a/wandb/run-20250101_112144-t9wzg2aq/logs/debug-internal.log" "b/wandb/run-20250101_112144-t9wzg2aq/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20250101_112144-t9wzg2aq/logs/debug-internal.log" @@ -0,0 +1,45074 @@ +2025-01-01 11:21:44,535 INFO StreamThr :2690175 [internal.py:wandb_internal():85] W&B internal server running at pid: 2690175, started at: 2025-01-01 11:21:44.534326 +2025-01-01 11:21:44,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:21:44,545 INFO WriterThread:2690175 [datastore.py:open_for_write():87] open: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/run-t9wzg2aq.wandb +2025-01-01 11:21:44,546 DEBUG SenderThread:2690175 [sender.py:send():379] send: header +2025-01-01 11:21:44,577 DEBUG SenderThread:2690175 [sender.py:send():379] send: run +2025-01-01 11:21:44,878 INFO SenderThread:2690175 [dir_watcher.py:__init__():211] watching files in: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files +2025-01-01 11:21:44,879 INFO SenderThread:2690175 [sender.py:_start_run_threads():1188] run started: t9wzg2aq with start time 1735730504.53736 +2025-01-01 11:21:44,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:21:44,903 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:21:44,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:21:45,003 DEBUG HandlerThread:2690175 [system_info.py:__init__():26] System info init +2025-01-01 11:21:45,003 DEBUG HandlerThread:2690175 [system_info.py:__init__():41] System info init done +2025-01-01 11:21:45,003 INFO HandlerThread:2690175 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:21:45,003 INFO SystemMonitor:2690175 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:21:45,004 INFO HandlerThread:2690175 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:21:45,004 INFO SystemMonitor:2690175 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:21:45,005 INFO SystemMonitor:2690175 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:21:45,005 INFO SystemMonitor:2690175 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:21:45,006 INFO SystemMonitor:2690175 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:21:45,007 INFO SystemMonitor:2690175 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:21:45,060 DEBUG HandlerThread:2690175 [system_info.py:probe():152] Probing system +2025-01-01 11:21:45,064 DEBUG HandlerThread:2690175 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:21:45,090 DEBUG HandlerThread:2690175 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:21:45,091 DEBUG HandlerThread:2690175 [system_info.py:probe():200] Probing system done +2025-01-01 11:21:45,091 DEBUG HandlerThread:2690175 [system_monitor.py:probe():223] {'os': 'Linux-6.8.0-48-generic-x86_64-with-glibc2.39', 'python': '3.10.0', 'heartbeatAt': '2025-01-01T11:21:45.060977', 'startedAt': '2025-01-01T11:21:44.520147', 'docker': None, 'cuda': None, 'args': ('--model_family', 'llama', '--apply_instruct_masks', '--token_scaled_loss', '--seq_parallel_size', '8', '--report_to', 'wandb', '--do_train', '--model_name_or_path', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--config_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--tokenizer_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--run_name', '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--output_dir', '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--config_overrides_json', '', '--gradient_accumulation_steps', '2', '--per_device_train_batch_size', '1', '--bf16', '--learning_rate', '1e-6', '--min_lr_ratio', '0.1', '--lr_scheduler_type', 'cosine', '--max_grad_norm', '1.0', '--adam_beta1', '0.9', '--adam_beta2', '0.95', '--weight_decay', '0.1', '--warmup_ratio', '0.05', '--optim', 'adamw_torch', '--logging_steps', '1', '--log_level', 'info', '--save_steps', '200', '--dataloader_num_workers', '1', '--disable_tqdm', 'true', '--use_fast_tokenizer', 'false', '--remove_unused_columns', 'false', '--ddp_find_unused_parameters', 'false', '--fsdp', 'auto_wrap offload', '--gradient_checkpointing', '--tokenized_mds_train', '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete', '--cuda_empty_cache', '--num_train_epochs', '1'), 'state': 'running', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'codePathLocal': 'finetune.py', 'codePath': 'prolong-final/finetune.py', 'git': {'remote': 'https://github.com/chtmp223/BookGen-dev.git', 'commit': '0e796521430a0f767be7c4dadba5c2fcaee1f909'}, 'email': 'ctpham@umass.edu', 'root': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev', 'host': 'gpu017', 'username': 'ctpham_umass_edu', 'executable': '/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10', 'cpu_count': 112, 'cpu_count_logical': 112, 'cpu_freq': {'current': 975.6290892857141, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 762.481, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2100.0, 'min': 800.0, 'max': 3800.0}, {'current': 785.34, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1300.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1103.96, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 790.077, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1587.993, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.494, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2547.505, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.425, 'min': 800.0, 'max': 3800.0}, {'current': 1997.889, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1300.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 782.826, 'min': 800.0, 'max': 3800.0}, {'current': 1100.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 768.56, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.299, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 782.047, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 794.129, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 438.487850189209, 'used': 16.457809448242188}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.3286399841309}} +2025-01-01 11:21:45,091 INFO HandlerThread:2690175 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:21:45,091 INFO HandlerThread:2690175 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:21:45,091 DEBUG HandlerThread:2690175 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2025-01-01 11:21:45,880 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/conda-environment.yaml +2025-01-01 11:21:51,118 DEBUG HandlerThread:2690175 [system_info.py:_save_conda():224] Saving conda packages done +2025-01-01 11:21:51,121 INFO HandlerThread:2690175 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:21:51,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:51,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:21:51,128 DEBUG SenderThread:2690175 [sender.py:send():379] send: files +2025-01-01 11:21:51,128 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:21:51,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:21:51,403 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:21:51,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:51,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:51,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:51,461 INFO wandb-upload_0:2690175 [upload_job.py:push():130] Uploaded file /tmp/tmp1xed_02cwandb/eeo9bmuh-wandb-metadata.json +2025-01-01 11:21:51,479 DEBUG SenderThread:2690175 [sender.py:send():379] send: telemetry +2025-01-01 11:21:51,480 DEBUG SenderThread:2690175 [sender.py:send():379] send: config +2025-01-01 11:21:51,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: telemetry +2025-01-01 11:21:51,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:21:51,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: telemetry +2025-01-01 11:21:51,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:21:51,482 WARNING SenderThread:2690175 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) +2025-01-01 11:21:51,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: telemetry +2025-01-01 11:21:51,483 DEBUG SenderThread:2690175 [sender.py:send():379] send: telemetry +2025-01-01 11:21:51,483 DEBUG SenderThread:2690175 [sender.py:send():379] send: config +2025-01-01 11:21:51,883 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/conda-environment.yaml +2025-01-01 11:21:51,883 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/requirements.txt +2025-01-01 11:21:51,884 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-metadata.json +2025-01-01 11:21:52,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:53,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:54,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:55,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:55,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:56,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:57,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:58,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:59,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:00,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:01,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:01,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:02,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:03,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:04,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:05,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:06,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:06,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:06,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:06,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:07,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:08,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:09,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:10,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:10,902 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:22:11,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:12,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:12,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:12,904 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:22:13,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:14,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:15,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:16,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:17,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:17,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:17,909 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/config.yaml +2025-01-01 11:22:18,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:19,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:20,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:21,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:21,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:21,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:22,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:23,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:23,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:24,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:25,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:26,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:27,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:28,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:29,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:29,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:30,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:31,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:32,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:33,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:34,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:34,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:35,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:36,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:36,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:36,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:37,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:38,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:22:38,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:22:38,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:22:38,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:22:38,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:22:38,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: metric +2025-01-01 11:22:38,144 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:22:38,144 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:22:38,146 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:22:38,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:38,937 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:22:38,938 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:22:39,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:40,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:40,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:41,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:42,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:43,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:44,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:45,007 DEBUG SystemMonitor:2690175 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:22:45,009 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:22:45,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:46,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:46,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:47,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:48,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:49,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:50,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:51,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:51,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:51,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:51,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:51,946 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/config.yaml +2025-01-01 11:22:52,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:53,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:54,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:55,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:56,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:57,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:57,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:58,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:59,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:00,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:01,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:02,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:03,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:03,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:04,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:05,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:06,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:06,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:06,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:07,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:08,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:08,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:09,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:10,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:11,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:12,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:13,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:23:13,273 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:23:13,273 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:23:13,278 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:23:13,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:13,969 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:23:14,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:14,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:14,970 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:23:15,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:23:15,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:16,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:17,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:18,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:19,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:19,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:20,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:21,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:21,409 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:21,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:22,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:23,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:24,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:24,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:25,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:26,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:27,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:28,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:29,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:30,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:30,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:31,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:32,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:33,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:34,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:35,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:35,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:36,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:36,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:36,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:37,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:38,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:39,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:40,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:40,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:41,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:42,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:43,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:44,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:45,022 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:23:45,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:46,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:46,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:47,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:48,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:23:48,065 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:23:48,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:23:48,067 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:23:48,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:49,004 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:23:49,005 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:23:49,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:50,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:51,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:51,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:51,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:51,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:52,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:53,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:54,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:55,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:56,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:56,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:57,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:58,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:59,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:00,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:01,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:02,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:02,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:03,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:04,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:05,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:06,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:06,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:06,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:07,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:07,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:08,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:09,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:10,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:11,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:12,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:13,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:13,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:14,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:15,029 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:24:15,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:16,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:17,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:18,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:18,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:19,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:24:19,420 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:24:19,421 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:24:19,422 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:24:19,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:20,037 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:24:20,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:21,037 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:24:21,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:21,409 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:21,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:22,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:23,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:23,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:24,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:25,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:26,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:27,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:28,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:29,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:29,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:30,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:31,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:32,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:33,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:34,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:34,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:35,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:36,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:36,409 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:36,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:37,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:38,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:39,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:39,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:40,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:41,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:42,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:43,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:44,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:45,025 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:24:45,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:45,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:46,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:47,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:48,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:49,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:50,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:50,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:51,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:24:51,206 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:24:51,206 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:24:51,208 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:24:51,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:51,623 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:51,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:52,067 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:24:52,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:53,068 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:24:53,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:54,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:55,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:55,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:56,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:57,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:58,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:59,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:00,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:01,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:01,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:02,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:03,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:04,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:05,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:06,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:06,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:06,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:06,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:07,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:08,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:09,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:10,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:11,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:12,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:12,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:13,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:14,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:15,027 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:25:15,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:16,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:17,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:17,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:18,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:19,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:20,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:21,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:25:21,263 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:25:21,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:21,265 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:21,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:21,674 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:21,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:22,098 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:25:22,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:22,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:23,099 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:25:23,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:24,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:25,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:26,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:27,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:28,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:28,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:29,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:30,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:31,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:32,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:33,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:33,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:34,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:35,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:36,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:36,668 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:36,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:37,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:38,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:38,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:39,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:40,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:41,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:42,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:43,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:44,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:44,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:45,031 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:25:45,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:46,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:47,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:48,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:49,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:49,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:50,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:51,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:51,671 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:51,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:52,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:53,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:25:53,600 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:25:53,601 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:53,602 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:54,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:54,129 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:25:55,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:55,130 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:25:55,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:56,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:57,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:58,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:59,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:01,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:02,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:03,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:04,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:05,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:06,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:06,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:06,672 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:06,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:07,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:08,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:09,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:10,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:11,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:12,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:12,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:13,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:14,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:15,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:15,035 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:26:16,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:17,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:18,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:18,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:19,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:20,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:21,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:21,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:21,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:22,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:23,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:23,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:24,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:24,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:26:24,597 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:26:24,597 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:24,599 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:25,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:25,166 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:26:25,167 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:26:26,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:27,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:28,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:29,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:29,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:30,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:31,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:32,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:33,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:34,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:35,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:35,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:36,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:36,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:36,706 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:37,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:38,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:39,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:40,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:40,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:41,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:42,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:43,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:44,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:45,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:45,039 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:26:46,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:46,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:47,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:48,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:49,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:50,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:51,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:51,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:51,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:51,711 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:52,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:53,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:54,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:54,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:26:54,479 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:26:54,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:54,481 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:55,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:55,193 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:26:55,194 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:26:56,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:56,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:57,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:58,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:59,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:00,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:01,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:02,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:02,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:03,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:04,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:05,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:06,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:06,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:06,708 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:07,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:07,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:08,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:09,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:10,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:11,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:12,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:13,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:13,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:14,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:15,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:27:15,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:16,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:17,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:18,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:18,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:19,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:20,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:21,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:21,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:21,709 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:22,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:22,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:22,126 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:27:22,127 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:22,128 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:22,225 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:27:23,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:23,226 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:27:24,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:24,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:25,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:26,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:27,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:28,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:29,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:29,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:30,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:31,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:32,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:33,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:34,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:36,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:36,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:36,709 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:37,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:38,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:39,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:40,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:40,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:41,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:42,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:43,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:44,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:45,046 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:27:45,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:46,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:46,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:47,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:48,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:49,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:50,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:51,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:51,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:51,160 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:27:51,160 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:51,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:51,161 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:51,250 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:27:51,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:51,710 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:52,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:53,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:53,252 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:27:54,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:55,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:56,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:56,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:57,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:58,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:59,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:00,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:01,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:01,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:02,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:03,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:04,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:05,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:06,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:06,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:06,710 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:06,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:07,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:08,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:09,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:10,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:11,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:12,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:12,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:13,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:14,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:15,050 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:28:15,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:16,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:17,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:17,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:18,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:19,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:20,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:21,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:21,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:21,717 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:22,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:22,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:23,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:23,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:28:23,582 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:28:23,583 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:28:23,584 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:24,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:24,289 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:28:25,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:25,290 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:28:26,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:27,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:28,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:28,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:29,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:30,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:31,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:32,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:33,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:34,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:34,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:35,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:36,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:36,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:36,714 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:37,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:38,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:39,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:39,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:40,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:41,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:42,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:43,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:44,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:45,054 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:28:45,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:45,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:46,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:47,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:48,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:49,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:50,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:50,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:51,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:51,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:51,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:52,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:53,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:53,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:28:53,543 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:28:53,543 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:28:53,544 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:54,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:54,319 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:28:55,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:55,320 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:28:56,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:56,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:57,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:58,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:59,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:00,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:01,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:01,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:02,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:03,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:04,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:05,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:06,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:06,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:06,718 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:06,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:07,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:08,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:09,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:10,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:11,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:12,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:12,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:13,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:14,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:15,058 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:29:15,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:16,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:17,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:18,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:18,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:19,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:20,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:21,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:21,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:21,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:22,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:23,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:23,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:24,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:25,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:26,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:26,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:29:26,570 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:29:26,571 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:29:26,572 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:29:27,350 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:29:27,351 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:29:27,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:28,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:29,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:29,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:30,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:31,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:32,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:33,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:34,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:35,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:35,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:36,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:36,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:36,721 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:37,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:38,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:39,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:40,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:41,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:41,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:42,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:43,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:44,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:45,063 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:29:45,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:46,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:47,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:47,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:48,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:49,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:50,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:51,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:51,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:51,721 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:52,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:52,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:53,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:54,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:55,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:56,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:57,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:58,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:58,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:58,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:29:58,804 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:29:58,805 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:29:58,807 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:29:59,381 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:29:59,381 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:29:59,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:00,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:01,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:02,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:03,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:03,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:04,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:05,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:06,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:06,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:06,721 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:07,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:08,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:09,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:09,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:10,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:11,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:12,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:13,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:14,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:14,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:15,066 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:30:15,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:16,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:17,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:18,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:19,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:20,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:20,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:21,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:21,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:21,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:22,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:23,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:24,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:25,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:25,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:26,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:27,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:30:27,425 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:30:27,426 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:30:27,427 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:30:28,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:28,413 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:30:29,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:29,414 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:30:30,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:31,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:31,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:32,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:33,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:34,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:35,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:36,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:36,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:36,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:36,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:37,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:38,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:39,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:40,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:41,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:42,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:42,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:43,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:44,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:45,070 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:30:45,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:46,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:47,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:48,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:48,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:49,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:50,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:51,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:51,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:51,730 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:52,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:53,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:53,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:54,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:55,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:55,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:30:55,991 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:30:55,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:30:55,994 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:30:56,439 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:30:56,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:57,440 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:30:57,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:58,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:59,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:59,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:00,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:01,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:02,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:03,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:04,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:05,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:05,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:06,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:06,726 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:06,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:07,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:08,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:09,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:10,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:10,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:11,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:12,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:13,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:14,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:15,074 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:31:15,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:16,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:16,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:17,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:18,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:19,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:20,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:21,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:21,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:21,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:21,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:22,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:23,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:24,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:31:24,456 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:31:24,456 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:24,457 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:24,462 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:31:24,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:25,463 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:31:25,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:26,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:26,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:27,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:28,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:29,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:30,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:31,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:32,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:32,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:33,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:34,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:35,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:36,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:36,731 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:36,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:37,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:37,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:38,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:39,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:40,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:41,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:42,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:43,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:43,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:44,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:45,078 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:31:45,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:46,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:47,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:48,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:49,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:49,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:50,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:51,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:51,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:51,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:52,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:53,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:54,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:54,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:55,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:56,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:57,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:31:57,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:31:57,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:57,014 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:57,505 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:31:57,506 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:31:58,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:59,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:00,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:00,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:01,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:02,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:03,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:04,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:05,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:05,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:06,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:06,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:06,732 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:07,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:08,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:09,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:10,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:11,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:11,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:12,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:13,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:14,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:15,082 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:32:15,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:16,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:17,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:17,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:18,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:19,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:20,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:21,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:21,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:21,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:22,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:22,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:23,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:24,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:25,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:26,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:27,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:28,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:28,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:29,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:30,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:31,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:32,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:32:32,040 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:32:32,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:32:32,041 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:32:32,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:32,545 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:32:33,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:33,546 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:32:34,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:34,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:35,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:36,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:36,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:36,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:37,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:38,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:39,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:40,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:40,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:41,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:42,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:43,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:44,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:45,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:32:45,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:46,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:46,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:47,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:48,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:49,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:50,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:51,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:51,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:51,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:51,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:52,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:53,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:54,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:55,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:56,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:56,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:57,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:58,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:59,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:00,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:01,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:02,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:02,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:02,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:33:02,617 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:33:02,618 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:02,619 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:03,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:03,582 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:33:03,583 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:33:04,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:05,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:06,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:06,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:06,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:07,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:07,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:08,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:09,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:10,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:11,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:12,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:13,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:13,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:14,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:15,089 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:33:15,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:16,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:17,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:18,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:18,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:19,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:20,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:21,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:21,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:21,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:22,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:23,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:23,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:24,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:25,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:26,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:27,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:28,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:28,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:29,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:30,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:31,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:31,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:33:31,975 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:33:31,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:31,977 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:32,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:32,614 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:33:33,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:33,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:33,615 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:33:34,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:35,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:36,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:36,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:36,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:37,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:38,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:39,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:39,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:40,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:41,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:42,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:43,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:44,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:44,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:45,093 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:33:45,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:46,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:47,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:48,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:49,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:50,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:50,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:51,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:51,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:51,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:52,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:53,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:54,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:55,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:55,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:56,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:57,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:58,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:59,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:00,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:00,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:01,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:01,499 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:34:01,499 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:01,501 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:01,659 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:34:01,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:02,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:03,660 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:34:03,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:04,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:05,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:06,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:06,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:06,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:06,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:07,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:08,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:09,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:10,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:11,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:12,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:12,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:13,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:14,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:15,097 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:34:15,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:16,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:17,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:18,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:18,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:19,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:20,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:21,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:21,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:21,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:22,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:23,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:23,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:24,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:25,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:26,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:27,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:28,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:29,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:29,180 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:34:29,181 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:29,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:29,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:29,689 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:34:29,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:34:29,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:30,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:31,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:32,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:33,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:34,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:34,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:35,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:36,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:36,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:36,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:37,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:38,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:39,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:40,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:40,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:41,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:42,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:43,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:44,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:45,101 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:34:45,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:46,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:46,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:47,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:48,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:49,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:50,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:51,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:51,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:51,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:51,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:52,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:53,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:54,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:55,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:56,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:57,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:57,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:58,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:59,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:00,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:01,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:02,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:03,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:35:03,543 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:35:03,543 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:35:03,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:03,545 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:35:03,721 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:35:03,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:04,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:05,723 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:35:05,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:06,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:06,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:06,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:07,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:08,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:08,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:09,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:10,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:11,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:12,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:13,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:13,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:14,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:15,106 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:35:15,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:16,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:17,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:18,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:18,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:19,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:20,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:21,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:21,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:21,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:22,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:23,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:24,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:24,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:25,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:26,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:27,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:28,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:29,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:30,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:30,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:31,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:32,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:33,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:34,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:35,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:35,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:36,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:36,735 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:36,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:36,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:35:36,983 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:35:36,983 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:35:36,985 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:35:37,759 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:35:37,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:35:37,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:38,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:39,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:40,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:41,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:41,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:42,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:43,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:44,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:45,109 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:35:45,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:46,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:47,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:47,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:48,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:49,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:50,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:51,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:51,735 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:51,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:52,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:52,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:53,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:54,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:55,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:56,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:57,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:58,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:58,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:59,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:00,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:01,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:02,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:03,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:03,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:04,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:05,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:06,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:06,736 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:06,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:07,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:36:07,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:36:07,642 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:36:07,643 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:36:07,793 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:36:08,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:08,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:09,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:09,795 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:36:10,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:11,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:12,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:13,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:14,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:14,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:15,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:15,113 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:36:16,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:17,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:18,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:19,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:19,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:20,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:21,737 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:22,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:23,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:24,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:24,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:25,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:26,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:27,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:28,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:29,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:30,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:30,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:31,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:32,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:33,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:34,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:35,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:35,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:36,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:36,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:36,740 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:37,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:38,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:39,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:40,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:40,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:41,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:41,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:36:41,346 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:36:41,346 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:36:41,348 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:36:41,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:36:41,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:36:42,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:43,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:44,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:45,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:45,116 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:36:46,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:46,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:47,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:48,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:49,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:50,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:51,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:51,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:51,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:51,740 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:52,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:53,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:54,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:55,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:56,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:56,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:57,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:58,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:59,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:00,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:01,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:01,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:02,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:03,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:04,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:05,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:06,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:06,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:06,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:06,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:07,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:08,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:09,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:10,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:11,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:12,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:12,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:13,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:13,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:37:13,783 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:37:13,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:13,785 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:13,860 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:37:14,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:15,120 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:37:15,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:15,862 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:37:16,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:17,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:18,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:18,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:19,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:20,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:21,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:21,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:21,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:22,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:23,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:23,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:24,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:25,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:26,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:27,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:28,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:28,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:29,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:30,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:31,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:32,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:33,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:35,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:36,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:36,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:36,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:37,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:38,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:39,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:41,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:42,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:43,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:44,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:44,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:37:44,425 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:37:44,425 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:44,426 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:44,897 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:37:45,125 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:37:45,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:46,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,898 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:37:47,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:48,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:49,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:50,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:51,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:51,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:51,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:51,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:52,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:53,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:54,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:55,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:56,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:57,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:57,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:58,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:59,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:00,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:01,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:02,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:02,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:03,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:04,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:05,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:06,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:06,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:06,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:07,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:07,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:08,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:09,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:10,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:11,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:12,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:13,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:13,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:14,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:15,130 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:38:15,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:15,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:38:15,459 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:38:15,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:15,461 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:15,928 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:38:16,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:16,929 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:38:17,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:18,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:18,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:19,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:20,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:21,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:21,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:21,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:22,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:24,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:25,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:26,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:27,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:28,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:28,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:29,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:30,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:31,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:32,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:33,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:34,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:34,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:35,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:36,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:36,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:36,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:37,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:38,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:39,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:40,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:40,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:41,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:42,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:43,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:44,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:45,136 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:38:45,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:46,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:46,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:46,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:38:46,961 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:38:46,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:46,964 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:47,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:47,963 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:38:48,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:48,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:38:49,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:50,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:51,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:51,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:51,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:51,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:52,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:53,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:54,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:55,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:56,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:56,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:57,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:58,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:59,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:00,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:01,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:02,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:02,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:03,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:04,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:05,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:06,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:06,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:06,742 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:07,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:07,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:08,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:09,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:10,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:11,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:12,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:13,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:13,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:14,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:15,135 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:39:15,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:16,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:17,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:18,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:18,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:18,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:39:18,741 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:39:18,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:18,743 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:19,005 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:39:19,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:20,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:21,006 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:39:21,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:21,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:21,742 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:22,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:23,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:24,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:24,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:25,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:26,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:27,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:28,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:29,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:30,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:30,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:31,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:32,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:33,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:34,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:35,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:35,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:36,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:36,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:36,742 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:37,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:38,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:39,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:40,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:41,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:41,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:42,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:43,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:44,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:45,139 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:39:45,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:46,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:47,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:47,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:48,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:49,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:50,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:51,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:51,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:51,745 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:52,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:52,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:53,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:39:53,197 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:39:53,198 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:53,203 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:53,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:54,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:39:54,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:55,046 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:39:55,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:56,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:57,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:58,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:58,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:59,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:00,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:01,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:02,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:03,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:04,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:04,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:05,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:06,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:06,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:06,745 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:07,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:08,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:09,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:09,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:10,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:11,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:12,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:13,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:14,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:14,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:15,144 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:40:15,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:16,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:17,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:18,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:19,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:19,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:20,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:20,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:20,856 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:40:20,856 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:20,858 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:21,078 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:40:21,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:21,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:21,745 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:22,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:23,080 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:40:23,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:24,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:24,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:25,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:26,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:27,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:28,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:29,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:30,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:30,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:31,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:32,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:33,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:34,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:35,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:35,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:36,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:36,746 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:36,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:37,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:38,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:39,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:40,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:41,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:41,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:42,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:43,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:44,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:45,148 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:40:45,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:46,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:47,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:47,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:48,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:49,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:50,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:51,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:51,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:51,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:52,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:52,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:53,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:53,459 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:40:53,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:53,462 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:53,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:54,110 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:40:54,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:55,111 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:40:55,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:56,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:57,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:58,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:58,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:59,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:00,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:01,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:02,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:03,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:04,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:04,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:05,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:06,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:06,751 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:06,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:07,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:08,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:09,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:09,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:10,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:11,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:12,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:13,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:14,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:15,152 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:41:15,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:15,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:16,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:17,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:18,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:19,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:20,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:20,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:21,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:21,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:21,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:22,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:23,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:24,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:25,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:25,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:26,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:27,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:28,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:41:28,438 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:41:28,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:41:28,439 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:41:28,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:29,151 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:41:29,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:30,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:31,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:41:31,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:31,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:32,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:33,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:34,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:35,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:36,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:36,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:36,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:36,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:37,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:38,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:39,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:40,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:41,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:42,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:42,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:43,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:44,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:45,156 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:41:45,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:46,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:47,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:48,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:48,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:49,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:50,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:51,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:51,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:51,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:52,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:53,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:53,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:54,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:55,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:56,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:57,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:58,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:58,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:59,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:41:59,878 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:41:59,878 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:41:59,880 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:00,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:42:00,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:01,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:42:01,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:02,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:03,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:04,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:04,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:05,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:06,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:06,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:06,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:07,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:08,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:09,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:09,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:10,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:11,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:12,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:13,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:14,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:14,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:15,160 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:42:15,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:16,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:17,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:18,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:19,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:20,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:20,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:21,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:21,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:21,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:22,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:23,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:24,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:26,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:27,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:28,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:29,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:30,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:30,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:31,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:31,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:42:31,652 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:42:31,652 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:42:31,654 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:32,216 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:42:32,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:33,217 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:42:33,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:34,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:35,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:36,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:36,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:36,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:36,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:37,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:38,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:39,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:40,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:41,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:42,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:42,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:43,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:44,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:45,164 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:42:45,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:46,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:47,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:48,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:48,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:49,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:50,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:51,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:51,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:51,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:52,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:53,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:53,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:54,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:55,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:56,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:57,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:58,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:58,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:59,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:00,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:01,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:02,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:02,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:43:02,637 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:43:02,638 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:02,640 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:03,249 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:43:03,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:04,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:04,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:05,251 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:43:05,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:06,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:06,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:06,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:07,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:08,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:09,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:10,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:10,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:11,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:12,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:13,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:14,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:15,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:43:15,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:16,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:16,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:17,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:18,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:19,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:20,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:21,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:21,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:21,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:21,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:22,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:23,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:24,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:25,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:26,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:27,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:27,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:28,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:29,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:30,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:31,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:32,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:32,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:43:32,586 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:43:32,586 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:32,588 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:33,286 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:43:33,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:33,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:34,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:35,287 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:43:35,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:36,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:36,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:36,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:37,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:38,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:38,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:39,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:40,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:41,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:42,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:43,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:43,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:44,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:45,172 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:43:45,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:46,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:47,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:48,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:49,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:49,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:50,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:51,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:52,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:53,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:54,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:56,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:57,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:58,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:59,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:00,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:00,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:01,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:02,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:03,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:03,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:44:03,997 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:44:03,998 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:04,000 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:04,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:44:04,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:05,316 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:44:05,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:06,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:06,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:06,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:06,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:07,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:08,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:09,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:10,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:11,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:11,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:12,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:13,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:14,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:15,176 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:44:15,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:16,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:17,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:17,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:18,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:19,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:20,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:21,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:21,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:21,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:22,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:22,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:23,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:24,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:25,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:26,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:27,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:28,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:28,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:29,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:30,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:31,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:32,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:44:33,196 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:44:33,197 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:33,198 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:33,344 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:44:33,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:34,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:35,346 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:44:35,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:36,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:36,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:36,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:37,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:38,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:39,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:39,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:40,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:41,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:42,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:43,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:44,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:45,180 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:44:45,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:45,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:46,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:47,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:48,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:49,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:51,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:51,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:51,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:52,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:53,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:54,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:55,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:56,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:56,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:57,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:58,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:59,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:00,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:01,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:01,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:02,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:03,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:04,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:05,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:05,290 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:45:05,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:05,293 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:05,375 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:45:05,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:06,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:06,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:07,377 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:45:07,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:08,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:09,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:10,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:11,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:12,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:12,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:13,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:14,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:15,183 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:45:15,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:16,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:17,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:18,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:18,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:19,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:20,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:21,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:22,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:23,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:23,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:24,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:25,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:26,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:27,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:28,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:28,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:29,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:30,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:31,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:32,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:33,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:34,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:34,970 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:45:34,971 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:34,972 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:35,407 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:45:35,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:36,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:36,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:36,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:37,409 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:45:37,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:38,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:39,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:39,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:40,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:41,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:42,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:43,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:44,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:45,187 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:45:45,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:45,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:46,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:47,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:48,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:49,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:50,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:50,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:51,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:51,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:51,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:52,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:53,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:54,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:55,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:56,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:56,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:57,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:58,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:59,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:00,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:01,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:01,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:02,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:03,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:03,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:46:03,892 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:46:03,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:46:03,894 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:46:04,445 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:46:04,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:05,446 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:46:05,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:06,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:46:06,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:46:06,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:06,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:07,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:08,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:09,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:10,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:11,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:12,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:12,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:13,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:14,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:15,192 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:46:16,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:16,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:17,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:17,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:18,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:19,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:20,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:21,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:46:21,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:46:21,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:22,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:22,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:23,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:24,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:25,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:26,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:27,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:28,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:28,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:29,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:30,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:31,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:32,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:46:32,711 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:46:32,711 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:46:32,712 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:46:33,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:33,475 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:46:33,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:34,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:35,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:35,476 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:46:36,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:36,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:46:36,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:46:37,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:38,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:38,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:39,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:40,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:41,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:42,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:43,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:44,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:44,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:45,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:45,195 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:46:46,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:47,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:48,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:49,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:49,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:50,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:51,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:51,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:46:51,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:46:52,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:53,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:54,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:55,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:46:55,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:56,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:57,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:58,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:46:59,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:00,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:00,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:01,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:02,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:03,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:04,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:04,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:47:04,684 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:47:04,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:47:04,686 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:47:05,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:05,508 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:47:05,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:06,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:06,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:47:06,754 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:47:07,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:07,510 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:47:08,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:09,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:10,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:11,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:11,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:12,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:13,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:14,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:15,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:15,199 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:47:16,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:16,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:17,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:18,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:19,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:20,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:21,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:21,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:47:21,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:47:21,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:22,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:23,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:24,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:25,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:26,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:27,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:27,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:28,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:29,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:30,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:31,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:32,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:32,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:33,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:34,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:35,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:36,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:47:36,137 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:47:36,137 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:47:36,139 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:47:36,541 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:47:36,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:36,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:47:36,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:47:37,542 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:47:37,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:37,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:38,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:39,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:40,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:41,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:42,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:43,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:43,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:44,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:45,203 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:47:45,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:46,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:47,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:48,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:48,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:49,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:50,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:51,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:51,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:47:51,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:47:52,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:53,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:53,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:54,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:55,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:56,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:57,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:58,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:47:59,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:47:59,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:00,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:01,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:02,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:03,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:04,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:04,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:05,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:05,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:48:05,708 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:48:05,709 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:48:05,710 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:48:06,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:06,576 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:48:06,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:48:06,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:48:07,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:07,577 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:48:08,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:09,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:10,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:10,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:11,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:12,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:13,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:14,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:15,207 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:48:15,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:15,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:16,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:17,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:18,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:19,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:20,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:21,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:21,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:21,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:48:21,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:48:22,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:23,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:24,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:25,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:26,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:27,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:27,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:28,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:29,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:30,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:31,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:32,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:32,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:33,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:34,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:35,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:48:35,228 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:48:35,229 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:48:35,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:48:35,606 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:48:35,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:36,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:36,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:48:36,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:48:37,608 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:48:37,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:37,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:38,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:39,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:40,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:41,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:42,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:43,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:43,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:44,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:45,212 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:48:45,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:46,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:47,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:48,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:48,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:49,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:50,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:51,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:51,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:48:51,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:48:52,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:53,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:53,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:54,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:55,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:56,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:57,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:58,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:48:59,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:48:59,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:00,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:01,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:02,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:03,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:04,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:04,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:05,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:06,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:06,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:49:06,757 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:49:07,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:49:07,604 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:49:07,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:49:07,607 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:49:07,636 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:49:08,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:08,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:09,638 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:49:09,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:10,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:10,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:11,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:12,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:13,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:14,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:15,215 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:49:15,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:15,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:16,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:17,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:18,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:19,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:20,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:21,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:21,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:21,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:49:21,758 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:49:22,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:23,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:24,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:25,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:26,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:26,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:27,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:28,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:29,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:30,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:31,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:31,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:32,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:33,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:34,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:35,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:36,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:36,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:49:36,425 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:49:36,425 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:49:36,427 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:49:36,669 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:49:36,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:36,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:49:36,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:49:37,670 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:49:37,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:38,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:39,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:40,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:41,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:42,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:42,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:43,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:44,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:45,218 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:49:45,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:46,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:47,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:47,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:48,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:49,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:50,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:51,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:49:51,832 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:49:51,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:52,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:52,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:53,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:54,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:55,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:56,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:57,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:58,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:49:58,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:49:59,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:00,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:01,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:02,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:03,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:03,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:04,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:05,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:06,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:50:06,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:50:06,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:07,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:08,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:50:08,536 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:50:08,536 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:50:08,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:08,537 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:50:08,701 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:50:08,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:09,702 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:50:09,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:10,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:11,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:12,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:13,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:14,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:14,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:15,222 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:50:15,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:16,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:17,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:18,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:19,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:20,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:20,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:21,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:50:21,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:50:21,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:22,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:23,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:24,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:25,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:26,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:26,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:27,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:28,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:29,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:30,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:31,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:31,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:32,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:33,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:34,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:35,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:36,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:50:36,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:50:36,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:37,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:37,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:50:37,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:50:37,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:50:37,228 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:50:37,735 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:50:37,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:38,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:39,737 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:50:39,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:40,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:41,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:42,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:42,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:43,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:44,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:45,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:50:45,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:46,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:47,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:47,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:48,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:49,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:50,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:51,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:50:51,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:50:51,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:52,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:52,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:53,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:54,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:55,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:56,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:57,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:58,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:50:58,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:50:59,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:00,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:01,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:02,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:03,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:03,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:04,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:05,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:06,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:51:06,840 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:51:06,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:06,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:51:06,997 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:51:06,997 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:51:06,999 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:51:07,770 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:51:07,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:08,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:09,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:09,772 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:51:09,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:10,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:11,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:12,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:13,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:14,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:14,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:15,230 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:51:15,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:16,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:17,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:19,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:19,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:20,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:21,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:21,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:51:21,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:51:22,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:23,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:24,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:25,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:25,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:26,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:27,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:28,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:29,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:30,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:30,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:31,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:32,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:33,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:34,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:35,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:51:35,009 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:51:35,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:51:35,011 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:51:35,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:35,803 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:51:35,803 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:51:36,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:36,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:36,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:51:36,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:51:37,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:38,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:39,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:40,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:41,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:41,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:42,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:43,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:44,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:45,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:45,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:51:46,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:47,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:47,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:48,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:49,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:50,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:51,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:51,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:51:51,840 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:51:52,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:52,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:53,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:54,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:55,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:56,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:57,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:58,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:51:58,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:51:59,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:00,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:01,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:02,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:03,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:03,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:04,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:05,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:06,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:06,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:52:06,760 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:52:06,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:52:06,762 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:52:06,836 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:52:07,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:07,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:07,180 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:07,838 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:52:08,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:08,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:09,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:10,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:11,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:12,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:13,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:14,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:14,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:15,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:15,239 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:52:16,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:17,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:18,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:19,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:19,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:20,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:21,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:21,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:21,912 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:22,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:23,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:24,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:25,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:25,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:26,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:27,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:28,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:29,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:30,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:30,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:31,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:32,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:33,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:34,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:35,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:35,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:36,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:36,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:36,916 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:37,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:38,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:39,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:39,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:52:39,698 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:52:39,698 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:52:39,700 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:52:39,885 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:52:40,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:40,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:41,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:41,887 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:52:42,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:43,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:44,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,243 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:52:46,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:46,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:47,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:48,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:49,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:50,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:51,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:51,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:51,955 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:52,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:52,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:53,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:54,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:55,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:56,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:57,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:57,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:58,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:59,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:00,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:01,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:02,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:03,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:03,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:04,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:05,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:06,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:06,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:06,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:07,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:08,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:09,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:09,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:09,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:53:09,726 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:53:09,726 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:53:09,728 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:53:09,918 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:53:10,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:11,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:11,920 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:53:12,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:13,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:14,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:14,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:15,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:15,247 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:53:16,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:17,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:18,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:19,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:19,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:20,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:21,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:21,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:21,954 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:22,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:23,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:24,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:24,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:25,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:26,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:27,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:28,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:29,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:30,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:30,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:31,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:32,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:33,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:34,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:35,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:35,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:36,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:36,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:36,958 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:37,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:38,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:38,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:53:38,770 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:53:38,770 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:53:38,772 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:53:38,947 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:53:39,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:39,947 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:53:40,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:41,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:41,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:42,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:43,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:44,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:45,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:45,251 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:53:46,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:46,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:47,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:48,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:49,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:50,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:51,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:51,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:51,959 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:52,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:52,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:53,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:54,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:55,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:56,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:57,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:57,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:58,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:59,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:00,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:01,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:02,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:03,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:03,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:04,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:05,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:06,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:06,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:06,965 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:07,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:08,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:09,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:09,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:54:09,156 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:54:09,157 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:09,158 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:09,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:09,978 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:54:09,978 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:54:10,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:11,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:12,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:13,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:14,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:14,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:15,256 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:54:15,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:16,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:17,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:18,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:19,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:20,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:20,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:21,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:21,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:21,964 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:22,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:23,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:24,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:25,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:25,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:26,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:27,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:28,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:29,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:30,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:31,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:31,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:32,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:33,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:34,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:35,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:36,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:36,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:36,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:36,972 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:37,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:38,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:38,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:54:38,683 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:54:38,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:38,685 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:39,010 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:54:39,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:40,011 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:54:40,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:41,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:42,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:42,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:43,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:44,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:45,259 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:54:45,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:46,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:47,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:48,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:48,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:49,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:50,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:51,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:51,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:51,968 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:52,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:53,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:54,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:54,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:55,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:56,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:57,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:58,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:59,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:59,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:00,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:01,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:02,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:03,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:04,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:04,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:05,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:06,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:06,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:06,974 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:07,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:08,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:08,208 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:55:08,208 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:08,210 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:08,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:09,050 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:55:09,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:10,051 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:55:10,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:10,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:11,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:12,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:13,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:14,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:15,262 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:55:15,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:16,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:16,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:17,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:18,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:19,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:20,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:21,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:21,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:21,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:21,974 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:22,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:23,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:24,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:25,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:26,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:26,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:27,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:28,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:29,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:30,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:31,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:32,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:32,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:33,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:34,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:35,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:36,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:36,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:36,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:37,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:38,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:38,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:39,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:40,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:41,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:42,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:43,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:43,172 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:55:43,173 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:43,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:43,175 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:43,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:44,096 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:55:44,097 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:55:44,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:45,266 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:55:45,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:46,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:47,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:48,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:48,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:49,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:50,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:51,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:51,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:51,981 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:52,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:53,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:54,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:54,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:55,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:56,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:57,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:58,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:59,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:59,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:00,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:01,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:02,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:03,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:04,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:05,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:05,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:06,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:06,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:06,997 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:07,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:08,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:09,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:10,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:10,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:11,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:12,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:13,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:56:13,172 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:56:13,173 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:13,175 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:13,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:14,136 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:56:14,137 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:56:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:15,270 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:56:15,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:16,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:16,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:17,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:18,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:19,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:20,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:21,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:21,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:21,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:21,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:22,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:23,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:24,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:25,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:26,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:26,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:27,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:28,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:29,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:30,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:31,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:32,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:32,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:33,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:34,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:35,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:36,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:36,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:36,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:37,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:38,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:38,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:39,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:40,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:41,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:42,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:43,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:43,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:43,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:56:43,675 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:56:43,675 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:43,676 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:44,181 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:56:44,181 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:56:44,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:45,274 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:56:45,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:46,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:47,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:48,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:49,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:49,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:50,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:51,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:51,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:51,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:52,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:53,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:54,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:54,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:55,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:56,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:57,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:58,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:59,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:00,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:00,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:01,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:02,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:03,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:04,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:05,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:05,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:06,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:06,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:06,990 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:07,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:08,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:09,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:10,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:10,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:11,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:12,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:12,368 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:57:12,368 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:12,370 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:12,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:13,211 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:57:13,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:14,212 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:57:14,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:15,278 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:57:15,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:16,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:16,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:17,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:18,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:19,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:20,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:21,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:21,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:21,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:21,990 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:22,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:23,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:24,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:25,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:26,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:27,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:27,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:28,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:29,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:30,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:31,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:32,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:32,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:33,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:34,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:35,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:36,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:36,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:36,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:37,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:38,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:38,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:39,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:40,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:41,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:42,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:43,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:43,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:44,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:44,336 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:57:44,337 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:44,339 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:44,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:45,246 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:57:45,282 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:57:45,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:46,247 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:57:46,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:47,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:48,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:49,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:49,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:50,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:51,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:51,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:51,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:52,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:53,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:54,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:54,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:55,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:56,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:57,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:58,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:59,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:59,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:00,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:01,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:02,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:03,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:04,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:05,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:05,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:06,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:06,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:06,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:07,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:08,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:09,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:10,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:10,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:11,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:12,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:13,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:14,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:15,283 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:58:15,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:15,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:58:15,860 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:58:15,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:58:15,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:15,866 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:58:16,272 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:58:16,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:17,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:18,274 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:58:18,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:19,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:20,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:21,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:21,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:21,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:21,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:22,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:23,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:24,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:25,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:26,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:27,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:27,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:28,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:29,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:30,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:31,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:32,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:32,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:33,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:34,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:35,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:36,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:36,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:36,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:37,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:38,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:38,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:39,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:40,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:41,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:42,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:43,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:43,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:44,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:58:44,837 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:58:44,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:58:44,839 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:58:45,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:45,288 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:58:45,296 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:58:45,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:46,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:58:46,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:47,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:48,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:49,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:49,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:50,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:51,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:51,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:51,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:52,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:53,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:54,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:54,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:55,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:56,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:57,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:58,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:59,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:00,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:00,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:01,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:02,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:03,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:04,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:05,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:06,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:06,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:07,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:07,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:07,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:08,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:09,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:10,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:11,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:11,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:12,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:59:12,917 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:59:12,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:59:12,920 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:59:13,328 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:59:13,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:14,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:14,330 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:59:15,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:15,292 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:59:16,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:18,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:19,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:20,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:21,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:21,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:21,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:22,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:23,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:23,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:24,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:25,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:26,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:27,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:28,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:28,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:29,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:30,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:31,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:32,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:33,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:33,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:34,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:35,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:36,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:36,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:36,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:37,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:38,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:39,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:39,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:40,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:41,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:42,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:59:43,330 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 11:59:43,330 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:59:43,333 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:59:43,361 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 11:59:44,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:44,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:44,362 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 11:59:45,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:45,297 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 11:59:46,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:47,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:48,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:49,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:50,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:50,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:51,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:51,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:51,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:52,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:53,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:54,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:55,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:55,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:56,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:57,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:58,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:59,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:00,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:00,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:01,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:02,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:03,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:04,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:05,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:06,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:06,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:07,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:08,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:09,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:10,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:11,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:11,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:12,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:13,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:14,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:15,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:15,306 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:00:15,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:15,822 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:00:15,822 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:15,823 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:16,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:16,399 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:00:16,399 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:00:16,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:17,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:18,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:19,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:20,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:21,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:21,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:21,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:22,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:22,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:23,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:24,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:25,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:26,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:27,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:27,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:28,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:29,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:30,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:31,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:33,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:34,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:35,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:36,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:36,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:36,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:37,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:38,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:38,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:39,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:40,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:41,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:42,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:44,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,304 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:00:45,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:45,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:00:45,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:45,795 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:46,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:46,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:00:46,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:00:47,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:48,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:48,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:49,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:50,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:51,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:51,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:51,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:52,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:53,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:54,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:54,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:55,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:56,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:57,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:58,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:59,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:59,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:00,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:01,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:02,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:03,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:04,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:05,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:05,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:06,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:06,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:06,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:07,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:08,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:09,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:10,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:10,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:11,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:12,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:13,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:14,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:15,306 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:01:15,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:15,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:01:15,857 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:01:15,857 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:15,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:15,859 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:16,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:16,459 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:01:16,460 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:01:17,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:18,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:19,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:20,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:21,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:21,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:21,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:21,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:22,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:23,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:24,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:25,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:26,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:27,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:27,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:28,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:29,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:30,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:31,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:32,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:32,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:33,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:34,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:35,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:36,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:36,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:36,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:37,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:38,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:38,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:39,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:40,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:41,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:42,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:44,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:45,317 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:01:45,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:46,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:46,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:01:46,907 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:01:46,907 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:46,909 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:47,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:47,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:01:48,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:48,493 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:01:48,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:49,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:50,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:51,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:51,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:51,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:52,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:53,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:54,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:54,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:55,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:56,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:57,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:58,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:59,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:59,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:00,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:01,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:02,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:03,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:04,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:04,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:05,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:06,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:07,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:08,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:09,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:10,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:10,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:11,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:12,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:13,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:14,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:15,321 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:02:15,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:15,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:15,579 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:02:15,580 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:15,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:15,582 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:16,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:16,528 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:02:16,529 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:02:17,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:18,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:19,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:20,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:21,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:21,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:21,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:21,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:22,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:23,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:24,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:25,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:26,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:27,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:27,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:28,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:29,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:30,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:31,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:32,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:32,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:33,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:34,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:35,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:36,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:36,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:36,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:37,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:38,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:38,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:39,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:40,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:41,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:42,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:43,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:43,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:44,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:45,332 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:02:45,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:46,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:47,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:48,126 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:02:48,126 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:48,128 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:48,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,561 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:02:48,562 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:02:49,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:49,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:50,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:51,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:51,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:51,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:52,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:53,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:54,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:54,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:55,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:56,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:57,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:58,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:59,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:00,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:00,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:01,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:02,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:03,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:04,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:05,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:05,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:03:06,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:03:07,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:08,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:09,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:10,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:11,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:11,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:12,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:13,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:14,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:15,325 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:03:15,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:16,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:17,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:17,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:18,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:19,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:20,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:21,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:03:21,484 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:03:21,485 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:03:21,490 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:21,592 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:03:21,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:21,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:03:21,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:03:22,593 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:03:22,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:23,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:23,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:24,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:25,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:26,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:27,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:28,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:28,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:29,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:30,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:31,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:32,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:33,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:33,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:34,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:35,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:36,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:36,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:03:36,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:03:37,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:38,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:39,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:39,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:40,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:41,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:42,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:43,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:44,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:44,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:45,334 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:03:45,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:46,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:47,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:48,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:49,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:49,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:50,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:03:50,799 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:03:50,799 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:03:50,801 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:51,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:51,618 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:03:51,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:51,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:03:51,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:03:52,619 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:03:52,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:53,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:54,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:54,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:55,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:56,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:57,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:58,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:59,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:00,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:00,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:01,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:02,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:03,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:04,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:05,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:05,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:06,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:06,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:04:06,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:04:07,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:08,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:09,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:10,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:11,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:11,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:12,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:13,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:14,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:15,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:04:15,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:16,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:17,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:17,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:18,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:19,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:20,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:20,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:04:20,995 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:04:20,995 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:04:20,997 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:04:21,650 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:04:21,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:21,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:04:21,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:04:22,651 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:04:22,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:23,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:23,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:24,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:25,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:26,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:27,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:28,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:28,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:29,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:30,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:31,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:32,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:34,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:34,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:34,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:35,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:36,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:36,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:04:36,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:04:37,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:38,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:39,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:39,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:40,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:41,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:42,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:43,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:44,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:45,339 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:04:45,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:45,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:46,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:47,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:48,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:49,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:50,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:50,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:51,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:04:51,865 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:04:51,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:04:51,867 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:04:52,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:52,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:04:52,285 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:04:52,683 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:04:52,683 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:04:53,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:54,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:55,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:55,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:04:56,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:57,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:58,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:04:59,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:00,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:00,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:01,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:02,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:03,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:04,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:05,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:06,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:06,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:07,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:07,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:05:07,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:05:08,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:09,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:10,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:11,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:11,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:12,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:13,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:14,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:15,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:15,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:05:16,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:17,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:17,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:18,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:19,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:20,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:21,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:22,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:22,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:05:22,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:05:22,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:23,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:24,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:25,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:25,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:05:25,698 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:05:25,698 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:05:25,703 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:05:25,726 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:05:26,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:26,728 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:05:27,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:27,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:28,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:29,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:30,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:31,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:32,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:33,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:33,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:34,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:35,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:36,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:37,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:37,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:05:37,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:05:38,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:39,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:39,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:40,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:41,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:42,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:43,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:44,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:44,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:45,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:45,347 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:05:46,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:47,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:48,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:49,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:50,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:50,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:51,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:52,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:52,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:05:52,293 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:05:53,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:54,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:55,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:55,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:05:56,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:56,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:05:56,591 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:05:56,591 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:05:56,597 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:05:56,752 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:05:57,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:58,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:05:58,754 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:05:59,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:00,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:01,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:01,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:02,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:03,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:04,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:05,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:06,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:07,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:06:07,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:06:07,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:07,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:08,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:09,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:10,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:11,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:12,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:12,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:13,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:14,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:15,351 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:06:15,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:16,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:17,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:18,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:18,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:19,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:20,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:21,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:22,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:06:22,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:06:22,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:23,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:23,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:24,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:25,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:26,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:27,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:28,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:28,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:29,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:06:29,561 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:06:29,561 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:06:29,566 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:06:29,782 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:06:29,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:30,783 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:06:30,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:31,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:32,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:33,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:33,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:34,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:35,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:36,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:37,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:06:37,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:06:37,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:38,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:39,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:39,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:40,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:41,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:42,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:43,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:44,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:44,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:45,355 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:06:45,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:46,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:47,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:48,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:49,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:50,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:50,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:51,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:52,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:06:52,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:06:52,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:53,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:54,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:55,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:06:55,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:56,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:57,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:58,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:06:59,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:06:59,491 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:06:59,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:06:59,493 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:06:59,813 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:06:59,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:00,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:00,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:01,815 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:07:01,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:02,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:03,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:04,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:05,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:06,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:06,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:07,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:07:07,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:07:07,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:08,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:09,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:10,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:11,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:11,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:12,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:13,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:14,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:15,358 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:07:15,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:16,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:17,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:17,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:18,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:19,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:20,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:21,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:22,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:07:22,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:07:22,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:23,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:23,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:24,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:25,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:26,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:27,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:28,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:28,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:29,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:30,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:31,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:32,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:33,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:33,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:34,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:07:34,460 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:07:34,461 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:07:34,463 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:07:34,848 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:07:34,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:35,849 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:07:35,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:36,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:37,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:07:37,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:07:37,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:38,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:39,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:39,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:40,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:41,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:42,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:43,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:44,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:44,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:45,363 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:07:45,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:46,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:47,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:48,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:49,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:49,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:50,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:51,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:52,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:07:52,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:07:52,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:53,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:54,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:55,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:07:55,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:56,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:57,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:58,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:07:59,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:00,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:00,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:01,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:02,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:03,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:04,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:05,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:08:05,175 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:08:05,175 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:08:05,181 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:08:05,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:08:05,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:06,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:06,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:07,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:08:07,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:08:07,888 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:08:07,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:08,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:09,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:10,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:11,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:11,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:12,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:13,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:14,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:15,366 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:08:15,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:16,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:17,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:17,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:18,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:19,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:20,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:21,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:22,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:08:22,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:08:22,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:22,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:23,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:24,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:25,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:26,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:27,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:27,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:28,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:29,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:30,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:32,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:32,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:33,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:34,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:35,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:36,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:37,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:37,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:08:37,292 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:08:37,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:08:37,445 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:08:37,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:08:37,448 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:08:37,917 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:08:38,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:38,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:39,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:39,918 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:08:40,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:41,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:42,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:43,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:43,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:44,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:45,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:45,370 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:08:46,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:47,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:48,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:49,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:49,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:50,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:51,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:52,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:52,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:08:52,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:08:53,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:54,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:54,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:08:55,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:56,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:57,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:58,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:59,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:08:59,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:00,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:01,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:02,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:03,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:04,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:04,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:05,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:06,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:06,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:09:06,837 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:09:06,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:09:06,840 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:09:06,949 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:09:07,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:07,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:09:07,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:09:08,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:09,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:09,951 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:09:10,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:10,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:11,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:12,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:13,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:14,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:15,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:15,374 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:09:16,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:16,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:17,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:18,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:19,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:20,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:21,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:21,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:22,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:22,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:09:22,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:09:23,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:24,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:25,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:26,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:27,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:27,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:28,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:29,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:30,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:31,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:32,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:32,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:33,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:34,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:35,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:36,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:36,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:09:36,756 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:09:36,757 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:09:36,759 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:09:36,984 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:09:37,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:37,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:09:37,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:09:37,986 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:09:38,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:38,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:39,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:40,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:41,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:42,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:43,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:43,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:44,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:45,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:45,378 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:09:46,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:47,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:48,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:49,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:49,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:50,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:51,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:52,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:52,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:09:52,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:09:53,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:54,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:54,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:09:55,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:56,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:57,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:58,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:59,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:09:59,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:00,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:01,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:02,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:03,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:04,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:04,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:05,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:06,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:06,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:10:06,805 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:10:06,805 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:10:06,807 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:10:07,014 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:10:07,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:07,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:10:07,292 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:10:08,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:09,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:10,016 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:10:10,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:10,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:11,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:12,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:13,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:14,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:15,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:15,382 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:10:16,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:16,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:17,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:18,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:19,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:20,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:21,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:21,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:22,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:22,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:10:22,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:10:23,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:24,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:25,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:26,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:27,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:27,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:28,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:29,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:30,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:31,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:32,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:32,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:33,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:34,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:35,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:36,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:37,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:10:37,082 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:10:37,083 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:10:37,084 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:10:37,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:10:37,502 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:10:37,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:38,051 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:10:38,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:38,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:39,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:40,053 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:10:40,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:41,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:42,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:43,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:43,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:44,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:45,386 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:10:45,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:46,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:47,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:48,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:49,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:49,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:50,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:51,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:52,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:10:52,500 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:10:52,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:53,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:54,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:54,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:10:55,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:56,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:57,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:58,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:59,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:10:59,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:00,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:01,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:02,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:03,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:04,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:05,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:05,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:06,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:07,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:11:07,510 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:11:07,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:08,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:09,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:11:09,399 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:11:09,400 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:11:09,401 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:11:09,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:10,089 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:11:10,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:10,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:11,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:12,090 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:11:12,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:13,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:14,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:15,390 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:11:15,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:16,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:16,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:17,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:18,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:19,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:20,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:21,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:21,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:22,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:11:22,501 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:11:22,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:23,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:24,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:25,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:26,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:26,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:27,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:28,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:29,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:30,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:31,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:32,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:32,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:33,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:34,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:35,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:36,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:37,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:11:37,511 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:11:37,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:38,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:38,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:39,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:40,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:41,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:11:41,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:11:41,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:11:41,485 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:11:41,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:42,121 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:11:42,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:43,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:44,123 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:11:44,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:44,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:45,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:11:45,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:46,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:47,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:48,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:49,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:49,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:50,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:51,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:52,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:11:52,503 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:11:53,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:54,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:54,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:11:55,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:56,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:57,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:58,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:11:59,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:00,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:00,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:01,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:02,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:03,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:04,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:05,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:05,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:06,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:07,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:07,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:12:07,508 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:12:08,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:09,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:10,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:10,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:12:10,837 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:12:10,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:12:10,839 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:12:11,150 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:12:11,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:11,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:12,151 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:12:12,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:13,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:14,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:15,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:15,398 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:12:16,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:17,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:17,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:18,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:19,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:20,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:21,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:22,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:22,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:12:22,505 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:12:22,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:23,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:24,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:25,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:26,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:27,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:27,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:28,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:29,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:30,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:31,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:32,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:32,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:33,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:34,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:35,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:36,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:37,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:37,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:12:37,508 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:12:38,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:38,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:39,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:39,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:12:39,544 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:12:39,544 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:12:39,546 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:12:40,182 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:12:40,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:41,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:42,184 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:12:42,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:43,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:44,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:44,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:45,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:45,402 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:12:46,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:47,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:48,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:49,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:49,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:50,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:51,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:52,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:52,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:12:52,509 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:12:53,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:54,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:54,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:12:55,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:56,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:57,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:58,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:12:59,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:00,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:00,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:01,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:02,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:03,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:04,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:05,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:05,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:06,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:07,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:13:07,184 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:13:07,184 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:13:07,186 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:13:07,212 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:13:07,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:13:07,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:07,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:13:08,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:09,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:10,214 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:13:10,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:11,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:11,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:12,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:13,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:14,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:15,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:15,407 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:13:16,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:16,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:17,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:18,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:19,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:20,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:21,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:22,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:22,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:22,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:13:22,604 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:13:23,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:24,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:25,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:26,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:27,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:28,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:28,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:29,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:30,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:31,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:32,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:33,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:33,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:34,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:35,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:36,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:37,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:37,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:13:37,588 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:13:37,588 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:13:37,590 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:13:38,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:13:38,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:13:38,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:38,254 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:13:38,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:39,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:40,256 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:13:40,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:41,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:42,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:43,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:44,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:44,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:45,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:45,410 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:13:46,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:47,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:48,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:49,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:49,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:50,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:51,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:52,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:52,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:13:52,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:13:53,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:54,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:54,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:13:55,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:56,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:57,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:58,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:13:59,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:00,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:00,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:01,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:02,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:03,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:04,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:05,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:05,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:06,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:06,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:14:06,926 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:14:06,926 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:14:06,928 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:14:07,292 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:14:07,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:07,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:14:07,734 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:14:08,293 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:14:08,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:09,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:10,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:11,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:11,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:12,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:13,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:14,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:15,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:15,414 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:14:16,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:16,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:17,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:18,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:19,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:20,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:21,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:22,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:22,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:22,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:14:22,733 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:14:23,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:24,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:25,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:26,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:27,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:27,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:28,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:29,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:30,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:31,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:32,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:33,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:33,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:34,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:35,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:36,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:37,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:37,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:14:37,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:14:38,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:38,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:39,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:14:39,223 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:14:39,224 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:14:39,226 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:14:39,324 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:14:39,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:40,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:41,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:42,326 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:14:42,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:43,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:44,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:44,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:45,418 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:14:45,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:46,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:47,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:48,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:49,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:49,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:50,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:51,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:52,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:52,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:14:52,739 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:14:53,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:54,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:54,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:14:55,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:56,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:57,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:58,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:14:59,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:00,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:00,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:01,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:02,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:03,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:04,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:05,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:05,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:06,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:07,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:07,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:15:07,742 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:15:08,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:15:08,524 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:15:08,525 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:15:08,526 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:15:08,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:09,358 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:15:09,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:10,360 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:15:10,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:11,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:11,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:12,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:13,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:14,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:15,422 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:15:15,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:16,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:16,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:17,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:18,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:19,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:20,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:21,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:22,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:22,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:22,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:15:22,739 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:15:23,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:24,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:25,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:26,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:27,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:27,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:28,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:29,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:30,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:31,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:32,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:33,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:33,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:34,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:35,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:36,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:37,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:37,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:15:37,744 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:15:38,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:38,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:39,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:40,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:40,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:15:40,889 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:15:40,889 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:15:40,890 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:15:41,386 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:15:41,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:42,386 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:15:42,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:43,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:43,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:44,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:45,427 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:15:45,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:46,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:47,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:48,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:49,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:49,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:50,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:51,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:52,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:52,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:15:52,744 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:15:53,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:54,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:54,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:15:55,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:56,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:57,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:58,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:15:59,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:00,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:00,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:01,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:02,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:03,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:04,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:05,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:06,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:06,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:07,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:07,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:16:07,748 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:16:08,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:09,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:10,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:11,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:11,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:12,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:12,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:16:12,807 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:16:12,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:16:12,809 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:16:13,424 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:16:13,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:14,425 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:16:14,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:15,430 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:16:15,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:16,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:16,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:17,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:18,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:19,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:20,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:21,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:22,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:22,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:22,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:16:22,745 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:16:23,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:24,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:25,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:26,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:27,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:27,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:28,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:29,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:30,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:31,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:32,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:33,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:33,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:34,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:35,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:36,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:37,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:37,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:16:37,748 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:16:38,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:38,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:39,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:40,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:41,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:42,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:16:42,479 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:16:42,479 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:16:42,480 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:16:42,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:43,453 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:16:43,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:44,454 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:16:44,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:44,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:45,434 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:16:45,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:46,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:47,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:48,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:49,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:50,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:50,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:51,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:52,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:16:52,748 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:16:52,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:53,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:54,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:55,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:16:55,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:56,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:57,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:58,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:16:59,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:00,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:01,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:01,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:02,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:03,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:04,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:05,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:06,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:06,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:07,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:17:07,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:17:07,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:08,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:09,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:10,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:11,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:11,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:17:11,893 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:17:11,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:17:11,895 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:17:12,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:12,487 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:17:13,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:14,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:14,489 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:17:15,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:15,437 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:17:16,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:16,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:17,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:18,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:19,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:20,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:21,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:22,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:22,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:22,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:17:22,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:17:23,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:24,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:25,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:26,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:27,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:27,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:28,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:29,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:30,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:31,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:32,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:33,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:33,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:34,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:35,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:36,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:37,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:37,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:17:37,757 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:17:38,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:38,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:39,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:40,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:41,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:42,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:42,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:17:42,421 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:17:42,421 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:17:42,423 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:17:42,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:17:43,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:44,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:44,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:44,523 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:17:45,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:45,441 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:17:46,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:47,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:48,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:49,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:50,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:50,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:51,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:52,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:52,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:17:52,755 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:17:53,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:54,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:55,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:56,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:17:56,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:57,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:58,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:17:59,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:00,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:01,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:01,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:02,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:03,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:04,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:05,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:06,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:07,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:07,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:07,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:18:07,769 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:18:08,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:09,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:10,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:11,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:12,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:12,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:13,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:13,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:18:13,620 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:18:13,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:18:13,622 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:18:14,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:14,554 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:18:15,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:15,445 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:18:16,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:16,556 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:18:17,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:17,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:18,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:19,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:20,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:21,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:22,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:22,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:18:22,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:18:22,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:23,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:24,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:25,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:26,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:27,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:28,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:28,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:29,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:30,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:31,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:32,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:33,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:33,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:34,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:35,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:36,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:37,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:37,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:18:37,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:18:38,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:38,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:39,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:40,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:41,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:42,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:43,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:44,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:44,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:45,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:45,451 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:18:45,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:18:45,710 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:18:45,711 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:18:45,713 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:18:46,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:46,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:18:47,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:48,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:48,586 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:18:49,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:49,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:50,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:51,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:52,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:52,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:18:52,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:18:53,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:54,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:54,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:18:55,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:56,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:57,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:58,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:18:59,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:00,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:00,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:01,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:02,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:03,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:04,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:05,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:06,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:06,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:07,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:07,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:19:07,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:19:08,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:09,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:10,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:11,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:11,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:12,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:13,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:14,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:14,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:19:14,920 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:19:14,920 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:19:14,922 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:19:15,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:15,454 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:19:15,616 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:19:16,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:16,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:16,617 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:19:17,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:18,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:19,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:20,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:21,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:22,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:22,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:22,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:19:22,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:19:23,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:24,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:25,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:26,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:27,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:28,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:28,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:29,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:30,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:31,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:32,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:33,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:34,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:34,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:35,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:36,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:37,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:37,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:19:37,772 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:19:38,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:39,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:39,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:40,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:41,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:42,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:43,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:43,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:19:43,783 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:19:43,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:19:43,785 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:19:44,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:44,652 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:19:45,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:45,458 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:19:45,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:46,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:46,654 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:19:47,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:48,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:49,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:50,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:50,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:51,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:52,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:52,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:19:52,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:19:53,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:54,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:55,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:56,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:19:56,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:57,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:58,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:19:59,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:00,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:01,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:01,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:02,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:03,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:04,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:05,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:06,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:07,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:07,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:07,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:20:07,759 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:20:08,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:09,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:10,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:11,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:12,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:12,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:13,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:14,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:15,462 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:20:15,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:16,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:20:16,390 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:20:16,391 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:20:16,392 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:20:16,685 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:20:16,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:17,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:17,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:18,687 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:20:18,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:19,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:20,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:21,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:22,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:20:22,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:20:22,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:22,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:23,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:24,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:25,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:26,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:27,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:28,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:28,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:29,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:30,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:31,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:32,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:33,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:33,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:34,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:35,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:36,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:37,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:20:37,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:20:37,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:38,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:38,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:39,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:40,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:41,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:42,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:43,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:44,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:44,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:45,472 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:20:45,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:46,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:47,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:48,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:49,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:20:49,104 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:20:49,104 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:20:49,106 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:20:49,723 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:20:49,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:50,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:50,724 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:20:50,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:51,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:52,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:20:52,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:20:52,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:53,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:54,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:55,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:20:55,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:56,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:57,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:58,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:20:59,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:00,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:01,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:01,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:02,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:03,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:04,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:05,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:06,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:06,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:07,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:21:07,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:21:07,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:08,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:09,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:10,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:11,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:12,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:12,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:13,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:14,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:15,471 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:21:15,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:16,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:17,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:17,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:18,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:19,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:21:19,116 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:21:19,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:21:19,118 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:21:19,753 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:21:19,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:20,754 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:21:20,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:21,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:22,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:21:22,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:21:22,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:22,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:23,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:24,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:25,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:26,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:27,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:28,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:28,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:29,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:30,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:31,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:32,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:33,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:33,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:34,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:35,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:36,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:37,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:21:37,760 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:21:37,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:38,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:38,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:39,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:40,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:41,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:42,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:43,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:44,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:44,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:45,475 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:21:45,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:46,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:47,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:48,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:48,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:21:48,975 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:21:48,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:21:48,977 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:21:49,783 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:21:49,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:50,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:50,784 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:21:50,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:51,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:52,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:21:52,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:21:52,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:53,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:54,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:55,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:21:55,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:56,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:57,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:58,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:21:59,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:00,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:00,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:01,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:02,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:03,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:04,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:05,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:05,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:06,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:07,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:22:07,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:22:07,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:08,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:09,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:10,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:11,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:11,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:12,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:13,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:14,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:15,476 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:22:15,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:16,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:16,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:17,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:18,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:19,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:22:19,549 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:22:19,549 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:22:19,551 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:22:19,817 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:22:19,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:20,818 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:22:20,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:21,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:22,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:22,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:22:22,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:22:22,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:23,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:24,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:25,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:26,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:27,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:27,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:28,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:29,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:31,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:32,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:33,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:33,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:34,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:35,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:36,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:37,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:37,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:22:37,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:22:38,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:38,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:39,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:40,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:41,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:42,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:43,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:44,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:44,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:45,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:45,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:22:46,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:47,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:22:47,013 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:22:47,013 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:22:47,014 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:22:47,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:47,843 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:22:48,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:48,844 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:22:49,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:50,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:50,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:51,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:52,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:52,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:22:52,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:22:53,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:54,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:55,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:55,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:22:56,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:57,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:58,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:22:59,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:00,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:00,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:01,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:02,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:03,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:04,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:05,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:06,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:06,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:07,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:07,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:23:07,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:23:08,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:09,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:10,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:11,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:11,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:12,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:13,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:14,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:15,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:15,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:23:15,422 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:23:15,423 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:23:15,425 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:23:15,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:23:15,869 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:23:16,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:16,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:16,870 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:23:17,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:18,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:19,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:20,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:21,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:22,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:22,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:22,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:23:22,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:23:23,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:24,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:25,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:26,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:27,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:28,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:28,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:29,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:30,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:31,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:32,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:33,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:33,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:34,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:35,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:36,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:37,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:37,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:23:37,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:23:38,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:38,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:39,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:40,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:41,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:42,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:43,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:44,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:44,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:45,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:45,494 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:23:46,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:47,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:48,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:48,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:23:48,998 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:23:48,998 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:23:49,000 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:23:49,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:49,904 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:23:50,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:50,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:50,905 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:23:51,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:52,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:52,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:23:52,762 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:23:53,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:54,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:55,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:23:55,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:56,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:57,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:58,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:23:59,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:00,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:01,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:01,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:02,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:03,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:04,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:05,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:06,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:06,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:07,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:07,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:24:07,762 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:24:08,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:09,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:10,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:11,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:11,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:12,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:13,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:14,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:15,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:15,496 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:24:16,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:16,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:17,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:18,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:19,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:20,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:21,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:21,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:24:21,962 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:24:21,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:24:21,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:21,969 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:24:22,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:22,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:24:22,762 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:24:22,934 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:24:22,935 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:24:23,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:24,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:25,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:26,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:27,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:27,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:28,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:29,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:30,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:31,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:32,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:33,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:33,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:34,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:35,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:36,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:37,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:37,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:24:37,762 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:24:38,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:38,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:39,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:40,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:41,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:42,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:43,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:44,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:44,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:45,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:45,500 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:24:46,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:47,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:48,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:49,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:49,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:50,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:51,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:52,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:52,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:24:52,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:24:52,485 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:24:52,491 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:24:52,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:24:52,902 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:24:52,962 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:24:53,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:54,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:54,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:24:55,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:24:55,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:56,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:57,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:58,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:24:59,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:00,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:00,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:01,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:02,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:03,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:04,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:05,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:06,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:06,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:07,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:07,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:25:07,898 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:25:08,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:09,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:10,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:11,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:11,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:12,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:13,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:14,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:15,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:15,504 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:25:16,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:16,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:17,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:18,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:19,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:20,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:21,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:22,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:22,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:22,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:25:22,903 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:25:23,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:24,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:25,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:25,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:25:25,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:25:25,793 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:25:25,799 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:25:26,000 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:25:26,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:27,001 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:25:27,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:27,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:28,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:29,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:30,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:31,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:32,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:33,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:33,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:34,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:35,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:36,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:37,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:37,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:25:37,901 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:25:38,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:39,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:39,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:40,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:41,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:42,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:43,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:44,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:44,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:45,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:45,508 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:25:46,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:47,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:48,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:49,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:49,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:50,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:51,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:52,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:52,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:25:52,903 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:25:53,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:54,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:55,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:25:55,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:56,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:57,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:58,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:25:58,231 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:25:58,231 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:25:58,237 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:25:58,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:25:59,034 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:25:59,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:00,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:00,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:01,036 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:26:01,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:02,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:03,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:04,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:05,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:05,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:06,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:07,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:07,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:26:07,904 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:26:08,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:09,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:10,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:11,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:11,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:12,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:13,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:14,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:15,512 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:26:15,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:16,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:16,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:17,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:18,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:19,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:20,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:21,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:22,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:22,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:22,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:26:22,907 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:26:23,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:24,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:25,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:26,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:27,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:27,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:28,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:29,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:26:29,294 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:26:29,294 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:26:29,296 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:26:29,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:30,069 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:26:30,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:31,070 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:26:31,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:32,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:33,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:33,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:34,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:35,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:36,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:37,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:37,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:26:37,905 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:26:38,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:39,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:39,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:40,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:41,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:42,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:43,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:44,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:44,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:45,517 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:26:45,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:46,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:47,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:48,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:49,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:49,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:50,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:51,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:52,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:52,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:26:52,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:26:53,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:54,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:55,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:26:55,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:56,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:57,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:58,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:26:59,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:00,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:00,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:00,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:27:00,714 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:27:00,715 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:27:00,720 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:27:01,107 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:27:01,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:02,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:03,109 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:27:03,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:04,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:05,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:06,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:06,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:07,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:07,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:27:07,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:27:08,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:09,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:10,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:11,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:12,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:12,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:13,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:14,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:15,520 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:27:15,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:16,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:17,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:17,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:18,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:19,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:20,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:21,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:22,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:22,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:27:22,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:27:23,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:23,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:24,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:25,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:26,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:27,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:28,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:28,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:29,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:30,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:27:30,610 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:27:30,610 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:27:30,611 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:27:31,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:31,154 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:27:31,154 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:27:32,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:33,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:33,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:34,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:35,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:36,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:37,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:37,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:27:37,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:27:38,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:39,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:39,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:40,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:41,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:42,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:43,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:44,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:44,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:45,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:45,524 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:27:46,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:47,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:48,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:49,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:49,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:50,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:51,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:52,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:52,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:27:52,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:27:53,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:54,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:55,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:55,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:27:56,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:57,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:58,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:58,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:27:58,505 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:27:58,506 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:27:58,508 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:27:59,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:27:59,184 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:27:59,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:28:00,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:00,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:01,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:02,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:03,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:04,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:05,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:06,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:06,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:07,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:07,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:28:07,912 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:28:08,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:09,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:10,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:11,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:12,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:12,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:13,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:14,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:15,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:15,528 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:28:16,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:17,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:17,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:18,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:19,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:20,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:21,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:22,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:22,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:28:22,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:28:23,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:23,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:24,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:25,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:26,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:27,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:28,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:28,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:29,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:30,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:31,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:28:31,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:28:31,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:28:31,013 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:28:31,212 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:28:31,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:32,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:33,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:33,213 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:28:34,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:34,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:35,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:36,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:37,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:37,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:28:37,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:28:38,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:39,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:39,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:40,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:41,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:42,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:43,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:44,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:45,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:45,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:45,531 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:28:46,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:47,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:48,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:49,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:50,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:51,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:51,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:52,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:52,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:28:52,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:28:53,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:54,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:55,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:56,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:57,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:57,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:28:58,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:28:59,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:00,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:01,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:02,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:03,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:03,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:29:03,373 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:29:03,374 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:29:03,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:03,375 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:29:04,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:04,244 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:29:05,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:05,245 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:29:06,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:07,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:07,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:29:07,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:29:08,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:08,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:09,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:10,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:11,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:12,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:13,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:14,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:14,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:15,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:15,535 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:29:16,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:17,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:18,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:19,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:19,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:20,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:21,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:22,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:22,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:29:22,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:29:23,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:24,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:25,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:25,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:26,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:27,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:28,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:29,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:30,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:30,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:31,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:32,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:33,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:34,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:34,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:29:34,798 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:29:34,798 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:29:34,800 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:29:35,273 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:29:35,274 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:29:35,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:35,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:36,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:37,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:37,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:29:37,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:29:38,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:39,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:40,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:41,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:41,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:42,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:43,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:44,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:45,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:45,540 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:29:46,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:46,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:47,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:48,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:49,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:50,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:51,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:52,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:52,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:52,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:29:52,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:29:53,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:54,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:55,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:56,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:57,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:57,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:29:58,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:29:59,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:00,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:01,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:02,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:02,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:03,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:04,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:05,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:05,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:30:05,470 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:30:05,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:30:05,472 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:30:06,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:06,305 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:30:07,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:07,306 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:30:07,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:07,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:30:07,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:30:08,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:09,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:10,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:11,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:12,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:13,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:13,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:14,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:15,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:15,543 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:30:16,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:17,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:18,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:18,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:19,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:20,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:21,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:22,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:22,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:30:22,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:30:23,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:24,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:24,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:25,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:26,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:27,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:28,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:29,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:29,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:30,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:31,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:32,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:33,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:34,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:34,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:35,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:36,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:36,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:30:36,802 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:30:36,803 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:30:36,804 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:30:37,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:37,332 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:30:37,333 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:30:37,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:30:37,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:30:38,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:39,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:40,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:40,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:41,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:42,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:43,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:44,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:45,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:45,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:45,548 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:30:46,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:47,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:48,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:49,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:50,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:50,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:51,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:52,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:52,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:30:52,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:30:53,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:54,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:55,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:56,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:30:56,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:57,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:58,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:30:59,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:00,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:01,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:02,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:02,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:03,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:04,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:05,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:06,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:07,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:07,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:07,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:31:07,760 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:31:07,761 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:31:07,762 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:31:08,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:31:08,174 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:31:08,364 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:31:08,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:09,365 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:31:09,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:10,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:11,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:12,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:12,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:13,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:14,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:15,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:15,552 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:31:16,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:17,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:17,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:18,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:19,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:20,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:21,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:22,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:23,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:31:23,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:31:23,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:23,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:24,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:25,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:26,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:27,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:28,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:28,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:29,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:30,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:31,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:32,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:33,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:33,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:34,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:35,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:31:35,223 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:31:35,223 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:31:35,225 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:31:35,393 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:31:35,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:36,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:37,394 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:31:37,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:38,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:31:38,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:31:38,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:39,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:39,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:40,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:41,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:42,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:43,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:44,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:44,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:45,556 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:31:45,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:46,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:47,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:48,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:49,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:50,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:50,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:51,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:52,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:53,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:31:53,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:31:53,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:54,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:55,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:56,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:31:56,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:57,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:58,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:31:59,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:00,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:01,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:01,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:02,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:03,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:04,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:05,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:32:05,295 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:32:05,296 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:32:05,298 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:32:05,421 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:32:05,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:06,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:06,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:07,422 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:32:07,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:08,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:32:08,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:32:08,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:09,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:10,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:11,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:12,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:12,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:13,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:14,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:15,560 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:32:15,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:16,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:17,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:18,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:18,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:19,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:20,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:21,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:22,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:23,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:32:23,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:32:23,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:24,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:24,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:25,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:26,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:27,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:28,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:29,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:29,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:30,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:31,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:32,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:33,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:34,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:32:34,292 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:32:34,292 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:32:34,294 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:32:34,452 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:32:34,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:35,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:35,453 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:32:35,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:36,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:37,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:38,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:32:38,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:32:38,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:39,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:40,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:40,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:41,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:42,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:43,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:44,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:45,564 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:32:45,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:45,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:46,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:47,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:48,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:49,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:50,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:51,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:51,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:52,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:53,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:32:53,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:32:53,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:54,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:55,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:56,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:32:56,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:57,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:58,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:32:59,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:00,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:01,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:01,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:02,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:03,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:04,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:05,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:06,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:33:06,798 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:33:06,799 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:33:06,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:06,800 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:33:07,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:07,488 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:33:07,489 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:33:08,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:33:08,173 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:33:08,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:09,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:10,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:11,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:12,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:12,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:13,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:14,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:15,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:15,567 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:33:16,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:17,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:18,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:18,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:19,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:20,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:21,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:22,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:23,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:33:23,172 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:33:23,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:24,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:24,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:25,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:26,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:27,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:28,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:29,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:29,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:30,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:31,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:32,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:33,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:34,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:34,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:35,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:36,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:37,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:37,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:33:37,260 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:33:37,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:33:37,262 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:33:37,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:33:37,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:33:38,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:33:38,172 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:33:38,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:39,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:40,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:40,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:41,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:42,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:43,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:44,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:45,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:45,571 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:33:45,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:46,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:47,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:48,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:49,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:50,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:50,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:51,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:52,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:53,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:33:53,172 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:33:53,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:54,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:55,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:56,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:56,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:33:57,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:58,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:33:59,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:00,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:01,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:02,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:02,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:03,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:04,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:05,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:06,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:07,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:07,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:08,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:34:08,172 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:34:08,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:34:08,260 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:34:08,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:34:08,262 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:34:08,565 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:34:08,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:09,566 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:34:09,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:10,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:11,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:12,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:13,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:13,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:14,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:15,575 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:34:15,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:16,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:17,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:18,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:19,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:19,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:20,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:21,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:22,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:23,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:34:23,172 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:34:23,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:24,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:25,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:25,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:26,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:27,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:28,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:29,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:30,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:30,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:31,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:32,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:33,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:34,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:35,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:35,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:36,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:37,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:38,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:34:38,180 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:34:38,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:39,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:34:39,629 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:34:39,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:34:39,631 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:34:40,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:40,602 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:34:41,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:41,603 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:34:41,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:42,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:43,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:44,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:45,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:45,580 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:34:46,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:47,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:47,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:48,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:49,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:50,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:51,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:52,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:53,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:53,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:34:53,177 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:34:53,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:54,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:55,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:56,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:57,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:58,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:34:58,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:34:59,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:00,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:02,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:03,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:03,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:04,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:05,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:06,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:07,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:08,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:08,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:35:08,177 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:35:09,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:09,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:10,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:11,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:11,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:35:11,503 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:35:11,503 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:35:11,504 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:35:11,635 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:35:12,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:13,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:13,636 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:35:14,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:14,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:15,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:15,583 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:35:16,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:17,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:18,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:19,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:19,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:20,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:21,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:22,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:23,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:23,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:35:23,177 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:35:24,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:25,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:25,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:26,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:27,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:28,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:29,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:30,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:30,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:31,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:32,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:33,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:34,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:35,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:35,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:36,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:37,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:38,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:38,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:35:38,188 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:35:39,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:40,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:40,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:41,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:41,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:35:41,633 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:35:41,634 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:35:41,635 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:35:41,669 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:35:42,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:43,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:43,671 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:35:44,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:45,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:45,587 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:35:45,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:46,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:47,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:48,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:49,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:50,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:50,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:51,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:52,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:53,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:53,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:35:53,181 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:35:54,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:55,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:55,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:35:56,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:57,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:58,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:35:59,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:00,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:00,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:01,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:02,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:03,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:04,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:05,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:05,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:06,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:07,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:08,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:08,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:36:08,193 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:36:09,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:10,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:10,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:36:10,560 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:36:10,561 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:36:10,562 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:36:10,703 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:36:10,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:11,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:11,704 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:36:12,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:13,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:14,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:15,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:15,591 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:36:16,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:16,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:17,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:18,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:19,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:20,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:21,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:21,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:22,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:23,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:23,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:36:23,187 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:36:24,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:25,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:26,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:26,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:27,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:28,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:29,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:30,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:31,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:32,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:32,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:33,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:34,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:35,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:36,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:37,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:37,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:38,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:38,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:36:38,194 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:36:39,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:40,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:41,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:41,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:36:41,459 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:36:41,459 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:36:41,461 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:36:41,730 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:36:42,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:42,731 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:36:43,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:43,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:44,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:45,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:45,595 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:36:46,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:47,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:48,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:48,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:49,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:50,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:51,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:52,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:53,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:53,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:36:53,189 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:36:54,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:54,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:36:55,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:56,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:57,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:58,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:59,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:36:59,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:00,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:01,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:02,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:03,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:04,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:05,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:05,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:06,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:07,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:08,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:08,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:37:08,194 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:37:09,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:10,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:10,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:11,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:12,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:13,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:13,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:37:13,817 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:37:13,817 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:37:13,818 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:37:14,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:14,758 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:37:15,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:15,599 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:37:15,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:16,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:16,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:37:17,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:18,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:19,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:20,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:21,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:21,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:22,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:23,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:37:23,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:37:23,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:24,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:25,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:26,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:27,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:27,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:28,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:29,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:30,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:31,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:32,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:32,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:33,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:34,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:35,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:36,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:37,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:38,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:37:38,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:37:38,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:38,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:39,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:40,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:41,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:42,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:43,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:43,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:44,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:44,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:37:44,822 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:37:44,823 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:37:44,824 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:37:45,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:45,603 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:37:45,787 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:37:46,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:46,788 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:37:47,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:48,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:49,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:49,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:50,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:51,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:52,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:53,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:37:53,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:37:53,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:54,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:55,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:55,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:37:56,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:57,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:58,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:37:59,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:00,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:00,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:01,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:02,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:03,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:04,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:05,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:06,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:06,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:07,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:08,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:38:08,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:38:08,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:09,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:10,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:11,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:11,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:12,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:13,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:14,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:15,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:15,607 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:38:16,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:16,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:38:16,507 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:38:16,508 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:38:16,510 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:38:16,819 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:38:17,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:17,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:18,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:18,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:38:19,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:20,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:21,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:22,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:22,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:23,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:38:23,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:38:23,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:24,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:25,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:26,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:27,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:28,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:28,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:29,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:30,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:31,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:32,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:33,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:33,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:34,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:35,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:36,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:37,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:38,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:38:38,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:38:38,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:39,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:39,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:40,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:41,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:42,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:43,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:44,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:45,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:45,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:45,609 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:38:46,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:46,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:38:46,776 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:38:46,777 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:38:46,779 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:38:46,847 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:38:47,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:48,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:48,849 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:38:49,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:50,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:50,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:51,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:52,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:53,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:38:53,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:38:53,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:54,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:55,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:55,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:38:56,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:57,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:58,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:38:59,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:00,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:00,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:01,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:02,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:03,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:04,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:05,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:05,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:06,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:07,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:08,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:39:08,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:39:08,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:09,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:10,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:11,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:11,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:12,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:13,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:14,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:15,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:15,615 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:39:16,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:16,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:17,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:18,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:19,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:20,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:20,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:39:20,358 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:39:20,359 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:39:20,360 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:39:20,879 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:39:21,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:21,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:22,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:22,880 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:39:23,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:39:23,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:39:23,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:24,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:25,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:26,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:26,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:27,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:28,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:29,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:30,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:31,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:32,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:32,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:33,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:34,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:35,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:36,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:37,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:37,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:38,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:39:38,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:39:38,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:39,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:40,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:41,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:42,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:43,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:43,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:44,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:45,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:45,620 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:39:46,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:47,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:48,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:49,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:49,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:50,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:50,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:39:50,787 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:39:50,787 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:39:50,789 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:39:50,906 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:39:51,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:52,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:52,907 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:39:53,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:39:53,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:39:53,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:54,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:54,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:39:55,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:56,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:57,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:58,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:39:59,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:00,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:00,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:01,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:02,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:03,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:04,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:05,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:06,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:06,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:07,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:08,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:40:08,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:40:08,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:09,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:10,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:11,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:12,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:12,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:13,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:14,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:15,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:15,628 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:40:16,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:17,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:17,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:18,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:19,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:20,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:40:20,148 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:40:20,148 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:40:20,150 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:40:20,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:20,936 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:40:21,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:22,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:22,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:22,938 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:40:23,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:40:23,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:40:23,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:24,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:25,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:26,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:27,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:28,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:28,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:29,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:30,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:31,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:32,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:33,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:34,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:34,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:35,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:36,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:37,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:38,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:40:38,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:40:38,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:39,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:39,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:40,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:41,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:42,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:43,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:44,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:45,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:45,629 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:40:45,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:46,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:47,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:48,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:49,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:50,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:50,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:51,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:52,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:53,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:40:53,197 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:40:53,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:40:53,355 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:40:53,355 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:40:53,357 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:40:53,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:53,963 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:40:54,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:54,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:40:55,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:40:55,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:56,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:57,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:58,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:40:59,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:00,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:01,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:01,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:02,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:03,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:04,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:05,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:06,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:06,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:07,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:08,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:41:08,197 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:41:08,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:09,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:10,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:11,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:12,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:12,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:13,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:14,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:15,633 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:41:15,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:16,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:17,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:18,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:18,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:19,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:20,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:21,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:22,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:23,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:41:23,208 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:41:23,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:23,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:24,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:25,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:25,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:41:25,786 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:41:25,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:41:25,788 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:41:25,992 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:41:26,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:26,993 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:41:27,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:28,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:29,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:29,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:30,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:31,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:32,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:33,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:34,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:35,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:35,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:36,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:37,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:38,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:41:38,203 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:41:38,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:39,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:40,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:40,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:41,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:42,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:43,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:44,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:45,636 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:41:45,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:45,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:46,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:47,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:48,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:49,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:50,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:51,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:51,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:52,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:53,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:41:53,203 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:41:53,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:54,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:41:54,168 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:41:54,169 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:41:54,171 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:41:54,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:55,020 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:41:55,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:56,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:57,022 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:41:57,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:41:57,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:58,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:41:59,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:00,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:01,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:02,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:03,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:03,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:04,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:05,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:06,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:07,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:08,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:42:08,203 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:42:08,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:08,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:09,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:10,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:11,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:12,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:13,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:14,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:14,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:15,640 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:42:15,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:16,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:17,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:18,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:19,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:20,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:20,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:21,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:22,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:23,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:42:23,205 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:42:23,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:24,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:25,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:25,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:26,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:42:26,020 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:42:26,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:42:26,022 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:42:26,061 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:42:27,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:27,062 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:42:28,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:29,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:30,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:31,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:31,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:32,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:33,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:34,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:35,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:36,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:37,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:37,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:38,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:38,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:42:38,206 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:42:39,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:40,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:41,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:42,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:42,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:43,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:44,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:45,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:45,644 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:42:46,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:47,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:48,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:48,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:49,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:50,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:51,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:52,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:53,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:53,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:42:53,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:42:53,208 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:42:54,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:55,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:55,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:42:55,309 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:42:55,310 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:42:55,311 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:42:56,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:56,099 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:42:57,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:57,099 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:42:58,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:59,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:42:59,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:00,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:01,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:02,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:03,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:04,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:05,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:05,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:06,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:07,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:08,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:08,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:43:08,384 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:43:09,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:10,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:10,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:11,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:12,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:13,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:14,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:15,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:15,648 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:43:15,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:16,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:17,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:18,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:19,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:20,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:21,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:21,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:22,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:23,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:23,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:43:23,287 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:43:24,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:25,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:25,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:43:25,709 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:43:25,709 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:43:25,710 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:43:26,125 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:43:26,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:26,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:27,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:27,128 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:43:28,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:29,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:30,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:31,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:32,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:32,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:33,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:34,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:35,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:36,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:37,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:38,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:38,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:38,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:43:38,502 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:43:39,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:40,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:41,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:42,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:43,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:43,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:44,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:45,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:45,652 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:43:46,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:47,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:48,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:49,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:49,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:50,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:51,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:52,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:53,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:53,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:43:53,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:43:54,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:54,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:43:55,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:56,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:56,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:43:56,636 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:43:56,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:43:56,639 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:43:57,158 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:43:57,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:58,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:59,160 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:43:59,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:43:59,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:00,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:01,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:02,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:03,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:04,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:05,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:05,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:06,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:07,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:08,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:44:08,406 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:44:08,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:09,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:10,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:11,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:11,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:12,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:13,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:14,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:15,657 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:44:15,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:16,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:16,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:17,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:18,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:19,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:20,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:21,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:21,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:22,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:23,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:44:23,406 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:44:23,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:24,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:25,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:26,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:27,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:44:27,362 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:44:27,362 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:44:27,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:27,363 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:44:28,190 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:44:28,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:29,191 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:44:29,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:30,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:31,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:32,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:32,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:33,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:34,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:35,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:36,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:37,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:38,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:44:38,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:44:38,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:38,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:39,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:40,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:41,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:42,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:43,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:44,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:44,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:45,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:45,661 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:44:46,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:47,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:48,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:49,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:50,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:50,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:51,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:52,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:53,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:44:53,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:44:53,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:54,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:55,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:55,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:44:56,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:57,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:58,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:44:59,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:00,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:01,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:01,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:02,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:03,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:45:03,191 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:45:03,192 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:45:03,193 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:45:03,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:45:03,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:04,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:05,222 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:45:05,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:06,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:06,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:07,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:08,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:45:08,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:45:08,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:09,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:10,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:11,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:12,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:12,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:13,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:14,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:15,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:15,665 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:45:16,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:17,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:17,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:18,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:19,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:20,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:21,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:22,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:23,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:45:23,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:45:23,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:23,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:24,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:25,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:26,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:27,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:28,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:28,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:29,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:30,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:31,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:32,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:33,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:33,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:34,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:45:34,506 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:45:34,507 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:45:34,508 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:45:34,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:35,258 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:45:35,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:36,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:37,259 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:45:37,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:38,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:45:38,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:45:38,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:38,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:39,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:40,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:41,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:42,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:43,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:43,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:44,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:45,669 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:45:45,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:46,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:47,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:48,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:48,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:49,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:50,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:51,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:52,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:53,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:45:53,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:45:53,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:54,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:45:54,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:55,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:56,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:57,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:58,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:45:59,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:00,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:00,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:01,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:02,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:46:02,929 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:46:02,930 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:46:02,931 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:46:03,282 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:46:03,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:04,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:05,284 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:46:05,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:05,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:06,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:07,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:08,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:08,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:46:08,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:46:09,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:10,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:11,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:11,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:12,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:13,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:14,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:15,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:15,673 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:46:16,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:16,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:17,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:18,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:19,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:20,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:21,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:22,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:22,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:23,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:23,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:46:23,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:46:24,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:25,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:26,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:27,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:28,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:28,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:29,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:30,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:31,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:32,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:33,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:34,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:34,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:35,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:36,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:37,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:38,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:38,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:46:38,419 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:46:39,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:46:39,130 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:46:39,130 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:46:39,132 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:46:39,313 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:46:39,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:40,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:40,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:41,315 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:46:41,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:42,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:43,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:44,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:45,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:45,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:45,677 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:46:46,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:47,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:48,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:49,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:50,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:51,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:51,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:52,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:53,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:46:53,414 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:46:53,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:54,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:55,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:56,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:57,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:57,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:46:58,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:46:59,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:00,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:01,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:02,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:02,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:03,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:04,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:05,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:06,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:07,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:07,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:08,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:47:08,419 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:47:08,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:09,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:10,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:10,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:47:10,857 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:47:10,858 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:47:10,859 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:47:11,345 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:47:11,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:12,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:12,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:13,347 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:47:13,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:14,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:15,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:15,681 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:47:16,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:17,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:18,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:18,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:19,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:20,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:21,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:22,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:23,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:47:23,420 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:47:23,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:24,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:24,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:25,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:26,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:27,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:28,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:29,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:29,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:30,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:31,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:32,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:33,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:34,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:34,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:35,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:36,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:37,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:38,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:47:38,420 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:47:38,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:38,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:47:38,943 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:47:38,943 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:47:38,945 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:47:39,368 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:47:39,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:39,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:40,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:41,370 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:47:41,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:42,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:43,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:44,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:45,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:45,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:45,685 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:47:46,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:47,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:48,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:49,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:50,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:51,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:51,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:52,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:53,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:47:53,420 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:47:53,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:54,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:55,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:56,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:56,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:47:57,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:58,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:47:59,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:00,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:01,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:01,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:02,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:03,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:04,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:05,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:06,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:06,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:07,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:08,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:48:08,422 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:48:08,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:09,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:48:09,355 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:48:09,356 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:48:09,357 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:48:09,399 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:48:09,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:10,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:11,401 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:48:11,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:12,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:12,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:13,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:14,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:15,689 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:48:15,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:16,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:17,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:17,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:18,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:19,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:20,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:21,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:22,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:23,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:48:23,421 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:48:23,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:23,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:24,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:25,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:26,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:27,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:28,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:28,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:29,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:30,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:31,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:32,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:33,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:34,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:34,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:35,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:36,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:37,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:38,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:48:38,424 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:48:38,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:39,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:40,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:48:40,322 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:48:40,323 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:48:40,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:40,324 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:48:40,432 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:48:40,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:41,433 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:48:41,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:42,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:43,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:44,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:45,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:45,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:48:45,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:46,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:47,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:48,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:49,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:50,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:51,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:51,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:52,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:53,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:48:53,424 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:48:53,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:54,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:55,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:56,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:57,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:48:57,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:58,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:48:59,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:00,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:01,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:02,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:03,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:03,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:04,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:06,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:07,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:08,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:08,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:49:08,428 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:49:08,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:09,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:10,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:10,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:49:10,091 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:49:10,091 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:49:10,092 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:49:10,464 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:49:11,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:11,465 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:49:12,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:13,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:13,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:14,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:15,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:15,696 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:49:16,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:17,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:18,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:19,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:19,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:20,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:21,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:22,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:23,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:23,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:49:23,428 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:49:24,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:24,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:25,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:26,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:27,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:28,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:29,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:29,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:30,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:31,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:32,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:33,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:34,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:35,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:35,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:36,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:37,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:38,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:38,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:49:38,432 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:49:39,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:40,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:40,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:41,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:41,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:49:41,164 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:49:41,165 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:49:41,166 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:49:41,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:49:42,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:43,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:43,493 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:49:44,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:45,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:45,699 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:49:45,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:46,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:47,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:48,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:49,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:50,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:51,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:51,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:52,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:53,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:53,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:49:53,434 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:49:54,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:55,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:56,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:56,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:49:57,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:58,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:49:59,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:00,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:01,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:02,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:02,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:03,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:04,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:05,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:06,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:07,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:07,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:08,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:08,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:50:08,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:50:09,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:10,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:11,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:12,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:12,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:50:12,072 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:50:12,073 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:50:12,074 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:50:12,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:50:13,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:13,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:13,522 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:50:14,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:15,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:15,703 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:50:16,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:17,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:18,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:18,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:19,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:20,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:21,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:22,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:23,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:23,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:50:23,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:50:24,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:24,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:25,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:26,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:27,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:28,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:29,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:29,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:30,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:31,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:32,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:33,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:34,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:35,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:35,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:36,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:37,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:38,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:38,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:50:38,437 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:50:39,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:40,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:40,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:41,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:42,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:43,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:43,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:50:43,951 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:50:43,951 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:50:43,953 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:50:44,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:44,559 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:50:45,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:45,560 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:50:45,708 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:50:45,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:46,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:47,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:48,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:49,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:50,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:51,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:51,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:52,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:53,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:53,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:50:53,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:50:54,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:55,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:56,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:57,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:57,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:50:58,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:50:59,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:00,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:01,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:02,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:02,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:03,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:04,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:05,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:06,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:07,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:08,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:08,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:51:08,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:51:08,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:09,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:10,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:11,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:12,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:13,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:13,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:14,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:15,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:15,711 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:51:16,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:17,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:17,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:51:17,233 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:51:17,234 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:51:17,235 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:51:17,593 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:51:18,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:18,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:19,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:19,594 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:51:20,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:21,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:22,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:23,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:23,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:51:23,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:51:24,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:24,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:25,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:26,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:27,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:28,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:29,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:30,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:30,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:31,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:32,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:33,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:34,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:35,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:35,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:36,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:37,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:38,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:38,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:51:38,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:51:39,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:40,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:41,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:41,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:42,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:43,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:44,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:45,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:45,719 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:51:46,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:46,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:47,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:48,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:48,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:51:48,894 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:51:48,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:51:48,896 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:51:49,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:49,621 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:51:50,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:51,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:51,622 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:51:51,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:52,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:53,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:53,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:51:53,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:51:54,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:55,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:56,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:57,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:57,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:51:58,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:51:59,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:00,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:01,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:02,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:02,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:03,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:04,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:05,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:06,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:07,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:08,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:08,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:52:08,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:52:08,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:09,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:10,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:11,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:12,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:13,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:13,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:14,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:15,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:15,720 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:52:16,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:17,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:18,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:18,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:19,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:20,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:21,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:21,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:52:21,767 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:52:21,768 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:52:21,769 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:52:22,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:22,651 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:52:23,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:23,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:52:23,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:52:23,652 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:52:23,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:24,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:25,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:26,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:27,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:28,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:28,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:29,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:30,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:31,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:32,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:33,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:33,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:34,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:35,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:36,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:37,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:38,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:38,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:52:38,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:52:39,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:39,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:40,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:41,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:42,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:43,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:44,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:44,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:45,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:45,731 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:52:46,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:47,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:48,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:49,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:52:49,273 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:52:49,273 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:52:49,275 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:52:49,677 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:52:49,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:50,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:50,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:51,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:51,679 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:52:52,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:53,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:53,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:52:53,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:52:54,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:55,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:55,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:52:56,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:57,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:58,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:52:59,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:00,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:00,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:01,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:02,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:03,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:04,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:05,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:06,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:06,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:07,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:08,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:08,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:53:08,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:53:09,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:10,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:11,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:11,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:12,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:13,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:14,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:15,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:15,729 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:53:16,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:16,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:17,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:17,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:53:17,435 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:53:17,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:53:17,441 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:53:17,705 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:53:18,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:19,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:19,706 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:53:20,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:21,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:22,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:22,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:23,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:23,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:53:23,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:53:24,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:25,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:26,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:27,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:28,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:28,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:29,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:30,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:31,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:32,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:33,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:33,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:34,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:35,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:36,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:37,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:38,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:38,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:53:38,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:53:39,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:39,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:40,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:41,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:42,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:43,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:44,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:44,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:45,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:45,741 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:53:46,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:47,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:47,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:53:47,661 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:53:47,661 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:53:47,664 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:53:47,731 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:53:48,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:49,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:49,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:49,733 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:53:50,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:51,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:52,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:53,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:53,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:53:53,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:53:54,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:55,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:55,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:53:56,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:57,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:58,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:53:59,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:00,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:00,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:01,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:02,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:03,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:04,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:05,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:05,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:06,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:07,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:08,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:08,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:54:08,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:54:09,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:10,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:11,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:11,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:12,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:13,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:14,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:15,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:15,739 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:54:16,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:16,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:17,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:18,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:19,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:54:19,348 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:54:19,349 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:54:19,351 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:54:19,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:19,761 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:54:20,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:21,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:21,763 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:54:22,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:22,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:23,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:54:23,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:54:23,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:24,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:25,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:26,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:27,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:27,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:28,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:29,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:30,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:31,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:32,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:33,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:33,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:34,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:35,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:36,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:37,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:38,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:54:38,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:54:38,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:39,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:39,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:40,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:41,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:42,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:43,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:44,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:44,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:45,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:45,742 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:54:46,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:47,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:48,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:49,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:49,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:50,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:51,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:52,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:54:52,101 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:54:52,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:54:52,104 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:54:52,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:52,806 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:54:53,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:54:53,440 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:54:53,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:53,807 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:54:54,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:55,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:55,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:54:56,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:57,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:58,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:54:59,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:00,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:01,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:01,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:02,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:03,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:04,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:05,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:06,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:07,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:07,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:08,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:55:08,440 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:55:08,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:09,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:10,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:11,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:12,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:13,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:13,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:14,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:15,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:15,746 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:55:16,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:17,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:18,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:18,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:19,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:20,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:21,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:22,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:23,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:55:23,440 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:55:23,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:55:23,605 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:55:23,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:55:23,611 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:55:23,837 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:55:23,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:23,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:24,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:25,839 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:55:25,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:26,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:27,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:28,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:29,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:29,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:30,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:31,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:32,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:33,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:34,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:35,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:35,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:37,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:38,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:38,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:55:38,440 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:55:39,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:40,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:41,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:41,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:42,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:43,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:44,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:45,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:45,750 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:55:46,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:47,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:47,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:48,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:49,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:50,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:51,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:52,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:52,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:53,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:53,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:55:53,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:55:54,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:54,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:55:54,777 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:55:54,777 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:55:54,780 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:55:54,868 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:55:55,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:55,869 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:55:56,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:57,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:58,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:55:58,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:55:59,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:00,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:01,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:02,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:03,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:03,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:04,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:05,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:06,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:07,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:08,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:08,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:56:08,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:56:08,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:09,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:10,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:11,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:12,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:13,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:13,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:14,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:15,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:15,754 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:56:16,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:17,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:18,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:19,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:19,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:20,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:21,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:22,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:23,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:23,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:56:23,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:56:24,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:24,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:25,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:26,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:27,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:27,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:56:27,300 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:56:27,301 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:56:27,306 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:56:27,903 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:56:28,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:29,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:29,905 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:56:30,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:30,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:31,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:32,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:33,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:34,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:35,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:36,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:36,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:37,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:38,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:38,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:56:38,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:56:39,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:40,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:41,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:41,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:42,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:43,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:44,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:45,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:45,758 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:56:46,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:47,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:47,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:48,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:49,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:50,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:51,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:52,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:52,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:53,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:53,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:56:53,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:56:54,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:55,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:56,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:57,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:57,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:56:58,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:58,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:56:58,881 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:56:58,881 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:56:58,884 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:56:58,940 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:56:59,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:56:59,941 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:57:00,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:01,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:02,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:03,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:03,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:04,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:05,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:06,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:07,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:08,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:08,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:57:08,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:57:08,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:09,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:10,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:11,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:12,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:13,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:14,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:14,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:15,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:15,762 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:57:16,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:17,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:18,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:19,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:20,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:20,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:21,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:22,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:23,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:23,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:57:23,457 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:57:24,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:25,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:25,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:26,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:27,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:28,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:29,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:29,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:57:29,938 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:57:29,939 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:57:29,941 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:57:29,973 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:57:30,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:30,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:31,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:31,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:57:32,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:33,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:34,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:35,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:36,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:36,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:37,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:38,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:38,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:57:38,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:57:39,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:40,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:41,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:42,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:42,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:43,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:44,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:45,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:45,766 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:57:46,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:47,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:47,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:48,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:49,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:50,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:51,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:52,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:53,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:53,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:53,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:57:53,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:57:54,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:55,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:56,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:57,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:58,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:57:58,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:57:59,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:00,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:01,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:02,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:02,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:58:02,946 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:58:02,946 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:58:02,952 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:58:03,007 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:58:03,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:03,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:04,008 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:58:04,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:05,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:06,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:07,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:08,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:08,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:58:08,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:58:09,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:09,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:10,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:11,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:12,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:13,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:14,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:14,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:15,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:15,770 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:58:16,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:17,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:18,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:19,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:19,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:20,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:21,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:22,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:23,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:58:23,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:58:23,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:24,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:25,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:25,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:26,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:27,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:28,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:29,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:30,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:30,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:31,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:32,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:58:32,332 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:58:32,332 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:58:32,336 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:58:32,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:33,039 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:58:33,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:34,040 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:58:34,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:35,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:36,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:36,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:37,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:38,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:58:38,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:58:38,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:39,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:40,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:41,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:42,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:42,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:43,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:44,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:45,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:45,774 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:58:46,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:47,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:47,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:48,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:49,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:50,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:51,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:52,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:53,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:53,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:58:53,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:58:53,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:54,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:55,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:56,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:57,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:58,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:58:59,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:58:59,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:00,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:01,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:02,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:03,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:04,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:04,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:59:04,459 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:59:04,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:59:04,465 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:59:04,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:05,071 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:59:05,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:06,072 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:59:06,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:07,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:08,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:59:08,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:59:08,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:09,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:10,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:10,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:11,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:12,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:13,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:14,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:15,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:15,779 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:59:15,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:16,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:17,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:18,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:19,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:20,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:20,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:21,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:22,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:23,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:59:23,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:59:23,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:24,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:25,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:26,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:26,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:27,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:28,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:29,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:30,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:31,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:31,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:32,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:33,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:59:33,617 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 12:59:33,618 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:59:33,619 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:59:34,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:34,101 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 12:59:34,101 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 12:59:35,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:36,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:37,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:37,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:38,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:38,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:59:38,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:59:39,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:40,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:41,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:42,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:43,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:43,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:44,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:45,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:45,782 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 12:59:46,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:47,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:48,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:48,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:49,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:50,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:51,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:52,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:53,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:53,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:59:53,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:59:54,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:54,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:55,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:56,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:57,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:58,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:59:59,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:59:59,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:00,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:01,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:02,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:03,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:04,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:04,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:04,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:00:04,692 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:00:04,692 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:00:04,693 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:00:05,129 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:00:05,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:06,130 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:00:06,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:07,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:08,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:08,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:00:08,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:00:09,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:09,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:10,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:11,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:12,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:13,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:14,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:14,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:15,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:15,787 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:00:16,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:17,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:18,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:19,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:19,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:20,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:21,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:22,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:23,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:23,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:00:23,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:00:24,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:25,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:25,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:26,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:27,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:28,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:29,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:30,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:30,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:31,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:32,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:33,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:34,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:35,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:35,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:36,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:00:36,063 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:00:36,063 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:00:36,065 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:00:36,162 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:00:36,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:37,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:38,163 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:00:38,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:38,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:00:38,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:00:39,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:40,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:41,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:41,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:42,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:43,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:44,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:45,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:45,791 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:00:46,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:46,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:47,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:48,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:49,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:50,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:51,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:52,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:52,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:53,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:53,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:00:53,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:00:54,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:55,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:56,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:57,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:58,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:00:58,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:00:59,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:00,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:01,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:02,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:03,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:03,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:04,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:05,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:06,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:01:06,134 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:01:06,134 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:01:06,136 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:01:06,198 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:01:06,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:07,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:08,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:08,199 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:01:08,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:01:08,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:01:08,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:09,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:10,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:11,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:12,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:13,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:14,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:14,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:15,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:15,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:01:16,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:17,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:18,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:19,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:19,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:20,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:21,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:22,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:23,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:01:23,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:01:23,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:24,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:25,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:25,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:26,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:27,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:28,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:29,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:30,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:30,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:31,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:32,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:33,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:34,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:34,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:01:34,644 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:01:34,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:01:34,645 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:01:35,224 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:01:35,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:35,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:36,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:37,226 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:01:37,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:38,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:01:38,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:01:38,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:39,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:40,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:41,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:41,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:42,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:43,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:44,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:45,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:45,797 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:01:46,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:46,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:47,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:48,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:49,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:50,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:51,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:52,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:52,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:53,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:01:53,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:01:53,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:54,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:55,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:56,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:57,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:58,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:01:58,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:01:59,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:00,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:01,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:02,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:03,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:03,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:04,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:05,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:02:05,353 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:02:05,353 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:02:05,354 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:02:05,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:06,257 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:02:06,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:07,258 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:02:07,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:08,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:08,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:02:08,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:02:08,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:09,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:10,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:11,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:12,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:13,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:14,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:14,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:15,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:15,801 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:02:16,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:17,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:18,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:19,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:19,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:20,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:21,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:22,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:23,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:02:23,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:02:23,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:24,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:25,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:25,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:26,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:27,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:28,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:29,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:30,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:30,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:31,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:32,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:33,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:34,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:35,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:02:35,043 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:02:35,043 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:02:35,044 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:02:35,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:35,285 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:02:35,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:36,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:37,287 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:02:37,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:38,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:02:38,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:02:38,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:39,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:40,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:41,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:41,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:42,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:43,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:44,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:45,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:45,805 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:02:46,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:46,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:47,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:48,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:49,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:50,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:51,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:52,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:52,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:53,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:02:53,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:02:53,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:54,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:55,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:56,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:57,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:58,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:02:58,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:02:59,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:00,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:01,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:02,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:03,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:03,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:04,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:03:04,654 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:03:04,654 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:03:04,656 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:03:05,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:05,320 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:03:06,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:07,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:07,322 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:03:08,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:08,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:03:08,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:03:08,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:09,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:10,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:11,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:12,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:13,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:14,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:14,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:15,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:15,809 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:03:16,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:17,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:18,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:19,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:19,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:20,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:21,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:22,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:23,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:23,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:03:23,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:03:24,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:24,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:25,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:26,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:27,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:28,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:29,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:30,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:30,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:31,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:32,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:33,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:34,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:35,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:35,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:36,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:37,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:38,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:38,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:03:38,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:03:38,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:03:38,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:03:38,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:03:38,795 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:03:39,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:39,357 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:03:40,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:40,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:41,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:41,359 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:03:42,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:43,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:44,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:45,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:45,814 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:03:45,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:46,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:47,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:48,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:49,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:50,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:51,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:51,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:52,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:53,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:53,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:03:53,454 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:03:54,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:55,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:56,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:03:56,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:57,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:58,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:03:59,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:00,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:01,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:01,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:02,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:03,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:04,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:05,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:06,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:06,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:07,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:08,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:08,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:04:08,455 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:04:09,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:09,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:04:09,474 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:04:09,474 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:04:09,476 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:04:10,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:10,389 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:04:11,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:11,389 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:04:11,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:12,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:13,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:14,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:15,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:15,818 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:04:16,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:16,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:17,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:18,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:19,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:20,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:21,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:22,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:22,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:23,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:23,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:04:23,454 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:04:24,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:25,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:26,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:27,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:28,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:28,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:29,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:30,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:31,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:32,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:33,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:34,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:34,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:35,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:36,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:37,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:38,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:38,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:04:38,459 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:04:39,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:39,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:40,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:40,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:04:40,969 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:04:40,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:04:40,974 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:04:41,420 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:04:41,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:42,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:43,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:43,421 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:04:44,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:44,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:45,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:45,822 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:04:46,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:47,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:48,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:49,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:50,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:50,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:51,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:52,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:53,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:53,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:04:53,457 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:04:54,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:55,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:55,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:04:56,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:57,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:58,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:04:59,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:00,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:01,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:01,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:02,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:03,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:04,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:05,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:06,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:07,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:07,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:08,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:08,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:05:08,461 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:05:09,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:10,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:11,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:11,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:05:11,489 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:05:11,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:05:11,492 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:05:12,450 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:05:12,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:12,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:13,451 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:05:13,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:14,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:15,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:15,825 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:05:16,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:17,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:17,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:18,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:19,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:20,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:21,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:22,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:23,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:23,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:05:23,461 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:05:23,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:24,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:25,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:26,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:27,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:28,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:28,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:29,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:30,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:31,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:32,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:33,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:34,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:34,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:35,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:37,466 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:05:38,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:39,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:05:39,155 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:05:40,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:45,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:45,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:45,829 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:05:47,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:50,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:05:54,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:54,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:05:54,700 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:05:55,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:05:55,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:00,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:04,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:05,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:06,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:11,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:13,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:15,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:06:15,232 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:06:15,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:15,833 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:06:16,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:21,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:22,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:23,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:24,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:27,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:32,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:37,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:37,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:06:37,187 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:06:37,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:37,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:38,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:39,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:39,685 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:06:40,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:41,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:42,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:42,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:43,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:44,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:45,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:45,838 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:06:46,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:06:46,209 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:06:46,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:47,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:48,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:48,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:49,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:50,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:51,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:52,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:53,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:53,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:54,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:55,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:56,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:57,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:58,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:06:58,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:06:59,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:00,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:01,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:07:01,209 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:07:01,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:02,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:03,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:03,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:04,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:05,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:06,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:07,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:08,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:08,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:09,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:10,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:11,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:12,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:13,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:13,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:14,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:15,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:15,843 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:07:16,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:07:16,210 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:07:16,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:17,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:18,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:19,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:19,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:20,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:21,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:22,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:23,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:24,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:25,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:25,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:26,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:27,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:28,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:29,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:30,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:30,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:31,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:07:31,209 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:07:31,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:32,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:36,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:41,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:45,846 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:07:46,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:51,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:52,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:07:52,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:52,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:07:53,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:54,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:55,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:56,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:57,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:07:57,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:58,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:07:59,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:00,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:01,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:02,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:02,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:03,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:04,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:05,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:06,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:07,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:07,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:08:07,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:08:07,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:08,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:09,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:10,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:11,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:12,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:13,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:13,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:14,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:15,850 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:08:15,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:16,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:17,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:18,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:19,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:19,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:20,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:21,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:22,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:22,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:08:22,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:08:23,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:24,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:25,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:25,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:26,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:27,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:28,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:29,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:30,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:30,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:31,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:32,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:33,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:34,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:35,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:36,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:36,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:37,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:37,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:08:37,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:08:38,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:39,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:40,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:41,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:41,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:42,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:43,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:44,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:45,854 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:08:45,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:46,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:47,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:47,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:48,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:49,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:50,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:51,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:52,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:52,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:08:52,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:08:52,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:53,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:54,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:55,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:56,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:57,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:58,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:08:58,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:08:59,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:00,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:01,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:02,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:03,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:04,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:04,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:05,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:06,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:07,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:09:07,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:09:07,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:08,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:09,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:10,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:10,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:11,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:12,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:13,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:14,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:15,858 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:09:15,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:15,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:16,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:17,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:18,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:19,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:20,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:20,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:21,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:22,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:09:22,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:09:22,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:23,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:24,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:25,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:25,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:26,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:27,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:28,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:29,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:30,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:31,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:31,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:32,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:33,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:34,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:35,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:36,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:37,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:37,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:09:37,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:09:37,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:38,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:39,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:40,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:41,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:42,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:42,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:43,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:44,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:45,862 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:09:45,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:46,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:47,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:48,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:48,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:49,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:50,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:51,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:52,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:09:52,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:09:52,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:53,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:09:53,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:09:58,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:03,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:09,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:14,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:15,871 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:10:19,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:24,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:29,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:10:29,366 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:10:29,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:30,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:30,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:31,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:32,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:33,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:10:33,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:34,221 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:10:34,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:35,222 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:10:35,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:35,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:36,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:37,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:38,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:39,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:40,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:41,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:41,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:42,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:43,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:44,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:10:44,362 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:10:44,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:45,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:45,870 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:10:46,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:46,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:47,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:48,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:49,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:50,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:51,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:52,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:52,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:53,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:54,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:55,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:56,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:57,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:10:57,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:58,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:10:59,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:10:59,364 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:10:59,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:00,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:01,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:01,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:11:01,731 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:11:01,731 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:11:01,736 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:11:02,241 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:11:02,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:02,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:03,242 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:11:03,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:04,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:05,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:06,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:07,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:08,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:08,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:09,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:10,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:11,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:12,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:13,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:13,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:14,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:14,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:11:14,580 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:11:15,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:15,875 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:11:16,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:17,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:18,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:18,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:19,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:20,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:21,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:22,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:23,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:24,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:24,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:25,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:26,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:27,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:28,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:29,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:29,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:11:29,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:29,588 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:11:30,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:31,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:32,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:11:32,037 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:11:32,038 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:11:32,040 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:11:32,272 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:11:32,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:33,273 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:11:33,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:34,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:35,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:35,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:36,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:37,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:38,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:39,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:40,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:41,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:41,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:42,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:43,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:44,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:11:44,579 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:11:44,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:45,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:45,879 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:11:46,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:46,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:47,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:48,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:49,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:50,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:51,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:52,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:52,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:53,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:54,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:55,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:56,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:57,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:11:57,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:58,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:11:59,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:11:59,586 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:11:59,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:00,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:01,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:02,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:02,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:03,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:03,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:12:03,919 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:12:03,920 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:12:03,924 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:12:04,300 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:12:04,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:05,301 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:12:05,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:06,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:07,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:08,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:08,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:09,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:10,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:11,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:12,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:13,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:14,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:14,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:12:14,584 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:12:14,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:15,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:15,881 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:12:16,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:17,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:18,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:19,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:19,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:20,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:21,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:22,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:23,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:24,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:25,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:25,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:26,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:27,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:28,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:29,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:12:29,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:29,604 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:12:30,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:30,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:31,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:32,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:33,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:34,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:35,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:12:35,113 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:12:35,114 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:12:35,116 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:12:35,336 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:12:35,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:36,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:36,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:37,338 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:12:37,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:38,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:39,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:40,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:41,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:41,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:42,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:43,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:44,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:12:44,590 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:12:44,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:45,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:45,885 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:12:46,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:46,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:47,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:48,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:49,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:50,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:51,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:52,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:52,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:53,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:54,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:55,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:56,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:57,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:58,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:12:58,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:12:59,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:12:59,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:12:59,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:00,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:01,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:02,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:03,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:03,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:04,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:13:04,117 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:13:04,117 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:13:04,118 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:13:04,365 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:13:04,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:05,365 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:13:05,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:06,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:07,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:08,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:09,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:09,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:10,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:11,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:12,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:13,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:14,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:13:14,596 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:13:14,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:14,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:15,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:15,890 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:13:16,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:17,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:18,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:19,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:20,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:20,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:21,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:22,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:23,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:24,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:25,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:26,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:26,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:27,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:28,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:29,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:13:29,598 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:13:29,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:30,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:31,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:31,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:32,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:33,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:13:33,274 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:13:33,274 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:13:33,276 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:13:33,402 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:13:33,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:34,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:35,404 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:13:35,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:36,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:37,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:37,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:38,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:39,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:40,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:41,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:42,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:43,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:43,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:44,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:13:44,597 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:13:44,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:45,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:45,892 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:13:46,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:47,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:48,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:48,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:49,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:50,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:51,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:52,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:53,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:54,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:54,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:55,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:56,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:57,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:58,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:13:59,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:13:59,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:13:59,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:13:59,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:00,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:01,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:02,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:03,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:14:03,702 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:14:03,703 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:14:03,704 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:14:04,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:04,439 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:14:05,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:05,440 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:14:05,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:06,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:07,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:08,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:09,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:10,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:11,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:11,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:12,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:13,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:14,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:14,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:14:14,600 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:14:15,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:15,896 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:14:16,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:16,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:17,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:18,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:19,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:20,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:21,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:22,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:22,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:23,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:24,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:25,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:26,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:27,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:27,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:28,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:29,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:29,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:14:29,607 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:14:30,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:31,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:32,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:32,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:33,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:33,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:14:33,412 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:14:33,413 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:14:33,414 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:14:33,469 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:14:34,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:35,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:35,471 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:14:36,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:37,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:38,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:38,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:39,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:40,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:41,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:42,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:43,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:44,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:44,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:44,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:14:44,604 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:14:45,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:45,897 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:14:46,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:47,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:48,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:49,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:49,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:50,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:51,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:52,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:53,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:54,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:55,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:55,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:14:56,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:57,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:58,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:59,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:14:59,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:14:59,618 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:15:00,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:00,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:01,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:02,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:03,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:03,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:15:03,841 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:15:03,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:15:03,843 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:15:04,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:04,505 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:15:05,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:05,507 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:15:06,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:06,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:07,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:08,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:09,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:10,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:11,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:11,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:12,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:13,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:14,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:14,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:15:14,607 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:15:15,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:15,902 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:15:16,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:16,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:17,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:18,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:19,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:20,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:21,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:22,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:22,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:23,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:24,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:25,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:26,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:27,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:27,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:28,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:29,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:29,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:15:29,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:15:30,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:31,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:32,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:32,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:33,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:33,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:15:33,472 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:15:33,473 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:15:33,474 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:15:33,536 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:15:34,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:35,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:35,537 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:15:36,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:37,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:38,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:38,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:39,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:40,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:41,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:42,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:43,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:44,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:44,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:15:44,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:15:44,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:45,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:45,906 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:15:46,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:47,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:48,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:49,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:50,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:50,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:51,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:52,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:53,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:54,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:55,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:56,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:56,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:15:57,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:58,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:59,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:15:59,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:15:59,615 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:16:00,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:01,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:01,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:02,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:03,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:04,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:04,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:16:04,867 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:16:04,867 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:16:04,869 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:16:05,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:05,564 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:16:05,564 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:16:06,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:06,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:07,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:08,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:09,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:10,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:11,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:12,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:12,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:13,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:14,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:14,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:16:14,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:16:15,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:15,910 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:16:16,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:17,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:17,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:18,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:19,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:20,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:21,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:22,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:23,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:23,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:24,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:25,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:26,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:27,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:28,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:29,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:29,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:16:29,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:16:29,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:30,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:31,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:32,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:33,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:34,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:35,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:35,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:36,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:37,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:16:37,109 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:16:37,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:16:37,114 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:16:37,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:37,594 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:16:37,594 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:16:38,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:39,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:40,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:41,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:41,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:42,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:43,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:44,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:44,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:16:44,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:16:45,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:45,914 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:16:46,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:46,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:47,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:48,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:49,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:50,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:51,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:52,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:52,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:53,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:54,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:55,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:56,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:57,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:58,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:58,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:16:59,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:16:59,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:16:59,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:17:00,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:01,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:02,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:03,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:03,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:04,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:05,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:06,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:07,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:08,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:09,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:17:09,122 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:17:09,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:17:09,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:09,126 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:17:09,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:09,623 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:17:09,624 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:17:10,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:11,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:12,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:13,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:14,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:14,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:17:14,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:17:14,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:15,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:15,918 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:17:16,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:17,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:18,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:19,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:20,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:20,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:21,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:22,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:23,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:24,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:25,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:25,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:26,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:27,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:28,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:29,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:29,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:17:29,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:17:30,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:30,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:31,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:32,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:33,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:34,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:35,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:36,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:36,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:37,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:38,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:38,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:17:38,655 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:17:38,656 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:17:38,657 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:17:39,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:39,654 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:17:40,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:40,655 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:17:41,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:42,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:42,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:43,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:44,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:44,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:17:44,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:17:45,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:45,922 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:17:46,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:47,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:47,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:48,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:49,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:50,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:51,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:52,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:53,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:53,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:54,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:55,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:56,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:57,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:58,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:58,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:17:59,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:17:59,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:17:59,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:18:00,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:01,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:02,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:03,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:03,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:04,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:05,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:06,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:07,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:08,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:09,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:09,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:10,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:11,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:18:11,144 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:18:11,144 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:18:11,146 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:18:11,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:11,683 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:18:12,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:12,684 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:18:13,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:14,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:14,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:18:14,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:18:14,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:15,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:15,927 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:18:16,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:17,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:18,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:19,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:20,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:20,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:21,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:22,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:23,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:24,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:25,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:25,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:26,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:27,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:28,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:29,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:29,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:18:29,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:18:30,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:30,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:31,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:32,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:33,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:34,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:35,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:36,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:36,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:37,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:38,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:39,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:40,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:41,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:41,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:41,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:18:41,707 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:18:41,708 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:18:41,711 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:18:41,715 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:18:42,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:43,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:44,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:44,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:18:44,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:18:44,717 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:18:45,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:45,930 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:18:46,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:46,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:47,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:48,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:49,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:50,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:51,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:52,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:52,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:53,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:54,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:55,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:56,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:57,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:57,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:18:58,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:59,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:18:59,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:18:59,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:19:00,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:01,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:02,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:02,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:03,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:04,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:05,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:06,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:07,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:08,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:08,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:09,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:10,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:11,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:11,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:19:11,895 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:19:11,896 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:19:11,899 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:19:12,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:12,739 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:19:13,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:13,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:14,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:19:14,614 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:19:14,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:14,740 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:19:15,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:15,935 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:19:16,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:17,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:18,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:18,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:19,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:20,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:21,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:22,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:23,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:24,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:24,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:25,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:27,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:27,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:28,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:29,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:19:29,614 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:19:29,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:29,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:30,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:31,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:32,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:33,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:34,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:35,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:35,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:36,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:37,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:38,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:39,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:40,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:40,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:41,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:42,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:43,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:44,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:19:44,623 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:19:44,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:45,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:45,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:45,942 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:19:46,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:19:46,636 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:19:46,636 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:19:46,637 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:19:46,773 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:19:47,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:48,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:48,774 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:19:49,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:50,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:51,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:51,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:52,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:53,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:54,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:55,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:56,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:57,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:57,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:19:58,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:59,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:19:59,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:19:59,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:20:00,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:02,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:02,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:03,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:04,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:05,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:06,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:07,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:08,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:08,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:09,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:10,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:11,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:12,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:13,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:13,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:14,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:14,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:20:14,623 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:20:15,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:15,950 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:20:16,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:17,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:17,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:20:17,845 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:20:17,845 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:20:17,846 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:20:18,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:18,804 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:20:18,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:19,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:20,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:20,805 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:20:21,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:22,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:23,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:24,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:24,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:25,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:26,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:27,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:28,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:29,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:29,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:20:29,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:20:29,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:30,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:31,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:32,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:33,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:34,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:35,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:35,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:36,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:37,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:38,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:39,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:40,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:40,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:41,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:42,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:43,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:44,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:44,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:20:44,624 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:20:45,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:45,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:45,953 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:20:46,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:47,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:47,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:20:47,577 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:20:47,578 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:20:47,583 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:20:47,834 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:20:48,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:48,835 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:20:49,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:50,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:51,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:51,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:52,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:53,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:54,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:55,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:56,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:57,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:57,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:20:58,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:59,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:20:59,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:20:59,624 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:00,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:01,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:02,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:02,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:03,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:04,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:05,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:06,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:07,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:08,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:08,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:09,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:10,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:11,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:12,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:13,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:13,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:14,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:14,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:15,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:15,963 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:21:16,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:17,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:18,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:21:18,208 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:21:18,208 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:21:18,210 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:21:18,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:18,865 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:21:19,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:19,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:20,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:20,866 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:21:21,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:22,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:23,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:24,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:24,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:25,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:26,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:27,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:28,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:29,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:29,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:29,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:29,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:30,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:31,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:32,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:33,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:34,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:35,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:35,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:36,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:37,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:38,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:39,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:40,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:41,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:41,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:42,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:43,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:44,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:44,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:44,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:45,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:45,957 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:21:46,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:46,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:47,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:48,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:49,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:50,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:50,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:21:50,826 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:21:50,826 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:21:50,829 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:21:50,904 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:21:51,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:52,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:52,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:52,905 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:21:53,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:54,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:55,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:56,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:57,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:58,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:58,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:59,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:59,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:59,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:00,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:01,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:02,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:03,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:03,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:04,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:05,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:06,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:07,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:08,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:09,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:09,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:10,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:11,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:12,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:13,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:14,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:14,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:14,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:14,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:15,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:15,962 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:22:16,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:17,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:18,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:19,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:20,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:20,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:20,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:22:20,937 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:22:20,938 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:22:20,942 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:22:21,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:21,935 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:22:22,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:22,936 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:22:23,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:24,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:25,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:26,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:26,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:27,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:28,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:29,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:29,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:29,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:30,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:31,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:31,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:32,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:33,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:34,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:35,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:36,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:37,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:37,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:38,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:39,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:40,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:41,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:42,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:43,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:43,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:44,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:44,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:44,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:45,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:45,978 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:22:46,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:47,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:48,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:48,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:49,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:50,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:51,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:51,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:22:51,728 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:22:51,728 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:22:51,734 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:22:51,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:22:52,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:52,965 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:22:53,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:54,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:54,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:55,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:56,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:57,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:58,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:59,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:59,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:59,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:59,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:00,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:01,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:02,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:03,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:04,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:05,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:05,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:06,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:07,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:08,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:09,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:10,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:12,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:13,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:14,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:14,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:14,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:15,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:15,968 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:23:16,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:16,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:17,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:18,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:19,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:20,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:20,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:23:20,595 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:23:20,596 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:23:20,597 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:23:20,991 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:23:21,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:22,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:22,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:22,993 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:23:23,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:24,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:25,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:26,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:27,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:27,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:28,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:29,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:29,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:29,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:30,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:31,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:33,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:34,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:35,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:36,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:37,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:38,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:38,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:39,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:40,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:41,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:42,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:43,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:43,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:44,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:44,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:44,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:45,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:45,972 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:23:46,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:47,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:48,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:48,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:49,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:50,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:51,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:52,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:53,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:23:53,277 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:23:53,278 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:23:53,280 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:23:53,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:54,025 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:23:54,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:54,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:55,026 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:23:55,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:56,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:57,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:58,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:59,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:59,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:59,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:59,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:00,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:01,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:02,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:03,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:04,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:05,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:05,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:06,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:07,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:08,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:09,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:10,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:10,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:11,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:12,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:13,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:14,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:14,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:14,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:15,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:15,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:15,975 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:24:16,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:17,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:18,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:19,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:20,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:21,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:21,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:22,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:22,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:24:22,855 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:24:22,855 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:22,857 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:23,061 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:24:23,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:24,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:25,063 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:24:25,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:26,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:27,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:27,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:28,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:29,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:29,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:29,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:30,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:31,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:33,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:34,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:35,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:36,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:37,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:38,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:38,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:39,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:40,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:41,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:42,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:43,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:44,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:44,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:44,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:44,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:45,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:45,984 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:24:46,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:47,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:48,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:49,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:50,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:50,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:51,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:52,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:24:52,102 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:24:52,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:52,104 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:52,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:53,090 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:24:53,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:54,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:55,092 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:24:55,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:56,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:56,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:57,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:58,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:59,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:59,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:59,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:00,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:01,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:01,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:02,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:03,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:04,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:05,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:06,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:07,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:07,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:08,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:09,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:10,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:11,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:12,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:13,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:13,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:14,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:14,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:14,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:15,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:15,982 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:25:16,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:17,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:18,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:18,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:19,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:20,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:21,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:25:21,333 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:25:21,333 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:21,334 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:21,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:22,124 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:25:22,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:23,125 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:25:23,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:24,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:24,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:25,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:26,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:27,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:28,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:29,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:29,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:29,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:29,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:30,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:31,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:32,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:33,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:34,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:35,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:35,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:36,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:37,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:38,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:39,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:40,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:41,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:41,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:42,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:43,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:44,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:44,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:44,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:45,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:45,995 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:25:46,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:47,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:47,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:48,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:49,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:50,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:51,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:51,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:25:51,863 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:25:51,863 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:51,864 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:52,155 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:25:52,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:52,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:53,157 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:25:53,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:54,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:55,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:56,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:57,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:58,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:58,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:59,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:59,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:59,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:00,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:01,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:02,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:03,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:03,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:04,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:05,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:06,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:07,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:08,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:09,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:09,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:10,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:11,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:12,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:13,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:14,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:14,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:14,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:14,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:15,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:15,990 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:26:16,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:17,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:18,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:19,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:20,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:20,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:21,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:22,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:23,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:24,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:26:24,019 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:26:24,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:24,024 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:24,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:26:24,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:25,187 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:26:25,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:26,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:26,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:27,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:28,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:29,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:29,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:29,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:30,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:31,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:31,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:32,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:33,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:34,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:35,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:36,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:37,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:37,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:38,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:39,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:40,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:41,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:42,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:42,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:43,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:44,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:44,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:44,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:45,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:45,995 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:26:46,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:47,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:47,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:48,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:49,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:50,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:51,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:52,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:53,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:53,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:54,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:26:54,111 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:26:54,111 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:54,115 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:54,218 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:26:54,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:55,219 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:26:55,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:56,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:57,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:58,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:58,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:59,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:59,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:59,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:00,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:01,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:02,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:03,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:03,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:04,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:05,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:06,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:07,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:08,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:09,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:09,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:10,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:11,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:12,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:13,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:14,634 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:14,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:14,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:15,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:15,998 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:27:16,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:17,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:18,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:19,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:20,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:20,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:21,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:22,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:22,303 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:27:22,303 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:22,305 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:22,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:23,256 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:27:23,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:24,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:25,258 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:27:25,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:26,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:26,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:27,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:28,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:29,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:29,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:29,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:30,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:31,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:31,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:32,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:33,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:34,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:35,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:36,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:37,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:37,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:38,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:39,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:40,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:41,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:42,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:43,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:43,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:44,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:44,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:44,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:45,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:46,003 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:27:46,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:47,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:48,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:49,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:49,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:50,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:51,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:52,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:52,200 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:27:52,200 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:52,202 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:52,283 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:27:52,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:53,284 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:27:53,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:54,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:54,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:55,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:56,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:57,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:58,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:59,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:59,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:59,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:59,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:00,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:01,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:02,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:03,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:04,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:05,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:05,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:06,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:07,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:08,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:09,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:10,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:10,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:11,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:12,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:13,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:28:14,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:28:14,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:15,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:15,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:16,007 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:28:16,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:17,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:18,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:19,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:20,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:21,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:21,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:22,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:23,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:28:23,406 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:28:23,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:28:23,408 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:28:23,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:24,313 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:28:24,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:25,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:28:25,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:26,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:26,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:27,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:28,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:29,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:28:29,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:28:29,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:30,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:31,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:32,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:32,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:33,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:35,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:35,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:36,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:37,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:38,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:38,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:39,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:40,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:41,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:42,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:43,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:43,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:44,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:28:44,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:28:44,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:45,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:46,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:28:46,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:47,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:48,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:49,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:49,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:50,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:51,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:52,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:28:52,460 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:28:52,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:28:52,461 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:28:52,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:53,345 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:28:53,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:54,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:54,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:55,346 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:28:55,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:56,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:57,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:58,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:28:59,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:28:59,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:28:59,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:28:59,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:00,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:01,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:02,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:03,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:04,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:05,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:05,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:06,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:07,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:08,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:09,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:10,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:10,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:11,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:12,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:13,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:14,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:29:14,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:29:14,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:15,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:15,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:16,015 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:29:16,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:17,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:18,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:19,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:20,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:21,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:21,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:22,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:29:22,046 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:29:22,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:29:22,048 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:29:22,374 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:29:22,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:23,374 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:29:23,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:24,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:25,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:26,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:26,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:27,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:28,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:29,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:29:29,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:29:29,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:30,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:31,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:31,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:32,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:33,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:34,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:35,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:36,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:37,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:37,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:38,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:39,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:40,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:41,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:42,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:43,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:43,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:44,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:29:44,633 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:29:44,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:45,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:46,019 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:29:46,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:47,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:48,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:49,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:49,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:50,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:51,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:52,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:53,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:54,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:29:54,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:55,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:29:55,414 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:29:55,415 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:29:55,416 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:29:55,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:56,401 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:29:56,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:57,402 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:29:57,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:58,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:29:59,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:29:59,633 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:29:59,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:00,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:00,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:01,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:02,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:03,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:04,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:05,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:05,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:06,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:07,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:08,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:09,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:10,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:11,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:11,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:12,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:13,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:14,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:30:14,633 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:30:14,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:15,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:16,023 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:30:16,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:17,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:17,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:18,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:19,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:20,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:21,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:22,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:22,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:23,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:24,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:25,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:27,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:27,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:27,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:29,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:29,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:30:29,635 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:30:30,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:30,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:30:30,186 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:30:30,186 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:30:30,188 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:30:30,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:30:31,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:31,431 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:30:32,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:33,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:33,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:34,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:35,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:36,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:37,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:38,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:38,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:39,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:40,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:41,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:42,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:43,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:44,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:44,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:30:44,636 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:30:44,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:45,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:46,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:46,027 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:30:47,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:48,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:49,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:50,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:50,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:51,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:52,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:53,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:54,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:55,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:56,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:56,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:30:57,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:58,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:59,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:30:59,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:30:59,639 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:31:00,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:01,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:01,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:02,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:02,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:31:02,406 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:31:02,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:31:02,408 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:31:02,466 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:31:03,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:03,467 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:31:04,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:05,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:06,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:07,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:07,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:08,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:09,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:10,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:11,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:12,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:13,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:13,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:14,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:14,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:31:14,640 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:31:15,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:16,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:16,031 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:31:17,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:18,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:19,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:19,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:20,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:21,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:22,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:23,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:24,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:24,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:25,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:26,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:27,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:28,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:29,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:29,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:31:29,640 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:31:29,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:30,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:30,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:31:30,616 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:31:30,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:31:30,618 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:31:31,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:31,498 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:31:32,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:33,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:33,500 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:31:34,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:35,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:35,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:36,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:37,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:38,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:39,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:40,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:41,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:41,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:42,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:43,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:44,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:44,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:31:44,640 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:31:45,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:46,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:46,036 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:31:47,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:47,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:48,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:49,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:50,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:51,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:52,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:52,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:53,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:54,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:55,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:56,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:57,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:57,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:31:58,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:59,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:31:59,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:31:59,646 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:32:00,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:00,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:32:00,213 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:32:00,213 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:32:00,215 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:32:00,528 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:32:01,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:01,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:32:02,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:03,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:03,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:04,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:05,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:06,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:07,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:08,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:08,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:09,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:10,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:11,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:12,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:13,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:14,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:14,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:32:14,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:32:14,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:15,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:16,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:16,040 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:32:17,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:18,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:19,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:20,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:20,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:21,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:22,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:23,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:24,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:25,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:25,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:26,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:27,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:28,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:28,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:32:28,849 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:32:28,850 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:32:28,851 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:32:29,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:29,561 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:32:29,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:32:29,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:32:30,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:30,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:31,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:31,562 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:32:32,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:33,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:34,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:35,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:36,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:36,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:37,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:38,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:39,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:40,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:41,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:42,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:42,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:43,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:44,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:44,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:32:44,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:32:45,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:46,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:32:46,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:47,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:48,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:48,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:49,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:50,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:51,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:52,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:53,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:53,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:54,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:55,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:56,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:57,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:58,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:58,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:32:59,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:32:59,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:32:59,623 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:32:59,623 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:32:59,624 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:33:00,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:33:00,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:33:00,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:00,597 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:33:01,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:01,598 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:33:02,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:03,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:04,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:04,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:05,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:06,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:07,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:08,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:09,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:10,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:10,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:11,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:12,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:13,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:14,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:15,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:33:15,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:33:15,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:15,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:16,048 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:33:16,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:17,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:18,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:19,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:20,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:21,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:21,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:22,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:23,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:24,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:25,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:26,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:27,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:27,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:28,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:33:28,258 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:33:28,259 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:33:28,261 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:33:28,625 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:33:28,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:29,626 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:33:29,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:30,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:33:30,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:33:30,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:31,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:32,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:32,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:33,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:34,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:35,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:36,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:37,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:38,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:38,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:39,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:40,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:41,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:42,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:43,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:43,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:44,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:45,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:33:45,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:33:45,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:46,052 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:33:46,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:47,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:48,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:49,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:49,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:50,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:51,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:52,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:53,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:54,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:33:54,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:55,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:56,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:57,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:57,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:33:57,707 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:33:57,707 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:33:57,708 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:33:58,656 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:33:58,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:59,657 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:33:59,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:33:59,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:00,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:34:00,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:34:00,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:01,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:02,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:03,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:04,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:05,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:05,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:06,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:07,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:08,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:09,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:10,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:10,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:11,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:12,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:13,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:14,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:15,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:34:15,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:34:15,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:15,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:16,057 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:34:16,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:17,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:18,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:19,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:20,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:21,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:21,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:22,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:23,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:24,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:25,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:26,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:26,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:27,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:27,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:34:27,701 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:34:27,701 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:34:27,703 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:34:28,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:28,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:34:29,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:29,691 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:34:30,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:34:30,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:34:30,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:31,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:31,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:32,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:33,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:34,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:35,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:36,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:37,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:37,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:38,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:39,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:40,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:41,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:42,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:42,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:43,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:44,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:45,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:34:45,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:34:45,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:46,059 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:34:46,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:47,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:48,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:48,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:49,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:50,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:51,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:52,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:53,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:53,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:54,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:55,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:56,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:57,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:34:57,333 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:34:57,333 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:34:57,334 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:34:57,720 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:34:57,720 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:34:58,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:34:58,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:34:59,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:00,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:00,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:00,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:01,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:02,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:03,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:04,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:04,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:05,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:06,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:07,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:08,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:09,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:09,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:10,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:11,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:12,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:13,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:14,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:15,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:15,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:15,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:15,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:16,063 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:35:16,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:17,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:18,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:19,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:20,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:21,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:21,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:22,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:23,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:24,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:25,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:26,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:27,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:27,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:28,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:29,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:30,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:30,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:30,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:31,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:32,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:32,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:35:32,377 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:35:32,378 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:35:32,381 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:35:32,756 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:35:32,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:33,758 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:35:33,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:34,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:35,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:36,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:37,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:38,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:38,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:39,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:40,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:41,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:42,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:43,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:44,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:44,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:45,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:45,042 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:45,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:46,067 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:35:46,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:47,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:48,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:49,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:50,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:50,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:51,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:52,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:53,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:54,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:55,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:55,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:56,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:57,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:58,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:59,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:00,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:00,050 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:00,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:00,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:01,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:02,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:03,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:04,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:36:04,487 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:36:04,487 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:36:04,488 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:36:04,791 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:36:04,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:05,792 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:36:05,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:06,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:06,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:07,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:08,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:09,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:10,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:11,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:12,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:12,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:13,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:14,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:15,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:15,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:15,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:16,071 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:36:16,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:17,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:18,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:18,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:19,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:20,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:21,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:22,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:23,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:23,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:24,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:25,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:26,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:27,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:28,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:29,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:29,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:30,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:30,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:30,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:31,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:32,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:33,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:34,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:34,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:35,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:36,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:36,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:36:36,985 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:36:36,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:36:36,987 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:36:37,823 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:36:37,824 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:36:37,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:38,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:39,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:40,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:40,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:41,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:42,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:43,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:44,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:45,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:45,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:45,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:45,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:46,075 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:36:46,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:47,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:48,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:49,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:50,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:51,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:51,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:52,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:53,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:54,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:55,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:56,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:57,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:57,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:58,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:59,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:00,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:00,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:00,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:01,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:02,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:02,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:03,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:04,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:05,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:06,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:37:06,099 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:37:06,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:37:06,101 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:37:06,855 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:37:06,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:07,856 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:37:07,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:08,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:08,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:09,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:10,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:11,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:12,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:13,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:14,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:14,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:15,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:15,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:15,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:16,081 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:37:16,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:17,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:19,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:19,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:20,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:20,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:21,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:22,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:23,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:24,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:25,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:25,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:26,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:27,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:28,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:29,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:30,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:30,049 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:30,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:30,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:31,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:32,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:33,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:34,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:37:34,637 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:37:34,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:37:34,638 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:37:34,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:37:35,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:35,887 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:37:36,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:36,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:37,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:38,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:39,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:40,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:41,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:42,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:42,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:43,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:44,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:45,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:45,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:45,050 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:46,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:46,084 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:37:47,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:48,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:48,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:49,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:50,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:51,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:52,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:53,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:53,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:54,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:55,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:56,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:57,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:58,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:58,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:59,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:00,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:00,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:00,056 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:01,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:02,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:03,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:03,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:04,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:05,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:05,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:38:05,154 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:38:05,154 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:38:05,155 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:38:05,912 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:38:05,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:38:06,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:07,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:08,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:09,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:09,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:10,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:11,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:12,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:13,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:14,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:15,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:15,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:15,053 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:15,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:16,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:16,087 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:38:17,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:18,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:19,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:20,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:21,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:21,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:22,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:23,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:24,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:25,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:26,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:27,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:27,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:28,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:29,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:30,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:30,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:30,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:31,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:32,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:32,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:33,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:33,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:38:33,590 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:38:33,590 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:38:33,591 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:38:33,937 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:38:33,938 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:38:34,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:35,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:36,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:37,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:38,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:38,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:39,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:40,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:41,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:42,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:43,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:44,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:44,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:45,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:45,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:45,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:46,091 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:38:46,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:47,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:48,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:49,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:50,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:50,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:51,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:52,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:53,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:54,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:55,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:55,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:56,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:57,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:58,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:59,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:00,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:00,073 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:00,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:00,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:01,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:02,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:03,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:04,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:04,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:39:04,770 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:39:04,771 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:04,772 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:04,965 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:39:05,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:05,966 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:39:06,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:06,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:07,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:08,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:09,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:10,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:11,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:12,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:12,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:13,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:14,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:15,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:15,063 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:15,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:16,095 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:39:16,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:17,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:18,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:18,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:19,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:20,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:21,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:22,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:23,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:23,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:24,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:25,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:26,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:27,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:28,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:29,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:29,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:30,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:30,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:30,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:31,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:32,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:33,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:34,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:34,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:35,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:36,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:36,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:39:36,565 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:39:36,566 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:36,568 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:36,998 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:39:37,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:38,000 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:39:38,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:39,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:39,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:40,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:41,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:42,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:43,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:44,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:45,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:45,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:45,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:45,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:46,099 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:39:46,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:47,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:48,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:49,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:50,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:50,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:51,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:52,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:53,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:54,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:55,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:55,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:56,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:57,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:58,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:59,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:00,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:00,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:00,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:00,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:01,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:02,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:03,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:04,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:05,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:06,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:06,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:07,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:08,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:08,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:40:08,663 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:40:08,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:40:08,665 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:40:09,027 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:40:09,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:10,028 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:40:10,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:11,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:12,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:12,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:13,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:14,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:15,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:15,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:15,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:16,102 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:40:16,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:17,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:18,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:18,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:19,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:20,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:21,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:22,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:23,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:23,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:24,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:25,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:26,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:27,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:28,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:28,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:29,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:30,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:30,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:30,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:31,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:32,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:33,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:33,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:34,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:35,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:36,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:37,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:38,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:38,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:38,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:40:38,620 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:40:38,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:40:38,621 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:40:39,058 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:40:39,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:40,060 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:40:40,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:41,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:42,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:43,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:44,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:44,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:45,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:45,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:45,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:46,107 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:40:46,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:47,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:48,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:49,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:50,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:50,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:51,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:52,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:53,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:54,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:55,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:55,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:56,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:57,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:58,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:59,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:00,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:00,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:00,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:01,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:01,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:02,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:03,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:04,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:05,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:06,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:06,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:07,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:07,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:41:07,823 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:41:07,823 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:07,825 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:08,088 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:41:08,088 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:41:08,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:09,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:10,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:11,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:12,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:12,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:13,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:14,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:15,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:15,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:15,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:16,112 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:41:16,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:17,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:18,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:18,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:19,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:20,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:21,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:22,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:23,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:23,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:24,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:25,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:26,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:27,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:28,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:28,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:29,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:30,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:30,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:30,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:31,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:32,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:33,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:33,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:34,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:35,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:36,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:37,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:38,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:38,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:38,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:41:38,949 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:41:38,950 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:38,951 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:39,131 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:41:39,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:40,132 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:41:40,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:41,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:42,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:43,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:44,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:44,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:45,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:45,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:45,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:46,115 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:41:46,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:47,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:48,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:49,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:50,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:50,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:51,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:52,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:53,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:54,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:55,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:55,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:56,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:57,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:58,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:59,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:00,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:00,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:00,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:01,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:01,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:02,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:03,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:04,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:05,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:06,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:06,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:07,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:08,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:09,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:10,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:10,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:10,632 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:42:10,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:10,634 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:11,160 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:42:11,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:11,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:12,161 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:42:12,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:13,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:14,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:15,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:15,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:15,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:16,119 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:42:16,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:17,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:17,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:18,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:19,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:20,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:21,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:22,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:22,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:23,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:24,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:25,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:26,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:27,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:27,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:28,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:29,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:30,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:30,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:30,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:31,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:32,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:33,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:33,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:34,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:35,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:36,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:37,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:38,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:38,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:39,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:40,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:41,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:41,370 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:42:41,370 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:41,372 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:41,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:42,188 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:42:42,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:43,189 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:42:43,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:43,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:44,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:45,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:45,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:45,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:46,123 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:42:46,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:47,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:48,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:49,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:49,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:50,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:51,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:52,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:53,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:54,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:54,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:55,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:56,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:57,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:58,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:59,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:59,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:00,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:43:00,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:43:00,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:01,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:02,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:03,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:04,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:05,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:05,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:06,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:07,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:08,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:09,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:10,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:11,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:11,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:12,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:13,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:43:13,199 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:43:13,200 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:43:13,201 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:43:13,217 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:43:13,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:14,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:15,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:43:15,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:43:15,219 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:43:15,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:16,127 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:43:16,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:17,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:17,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:18,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:19,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:20,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:21,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:22,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:22,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:23,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:24,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:25,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:26,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:27,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:28,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:28,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:29,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:30,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:43:30,066 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:43:30,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:31,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:32,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:33,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:33,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:34,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:35,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:36,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:37,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:38,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:39,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:39,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:40,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:41,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:42,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:43,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:44,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:44,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:43:44,398 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:43:44,399 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:43:44,400 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:43:44,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:45,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:43:45,066 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:43:45,257 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:43:45,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:46,131 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:43:46,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:47,259 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:43:47,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:48,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:49,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:50,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:50,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:51,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:52,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:53,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:54,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:55,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:43:55,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:56,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:58,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:58,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:43:59,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:00,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:44:00,066 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:44:00,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:01,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:01,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:02,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:03,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:04,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:05,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:06,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:06,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:07,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:08,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:09,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:10,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:11,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:11,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:12,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:13,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:14,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:15,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:44:15,069 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:44:15,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:44:15,792 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:44:15,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:44:15,793 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:44:16,134 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:44:16,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:16,288 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:44:17,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:17,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:17,289 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:44:18,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:19,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:20,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:21,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:22,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:22,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:23,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:24,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:25,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:26,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:27,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:28,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:28,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:29,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:30,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:44:30,067 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:44:30,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:31,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:32,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:33,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:33,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:34,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:35,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:36,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:37,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:38,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:39,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:39,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:40,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:41,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:42,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:43,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:44,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:44,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:45,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:44:45,070 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:44:45,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:45,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:44:45,509 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:44:45,509 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:44:45,511 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:44:46,138 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:44:46,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:46,319 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:44:47,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:47,320 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:44:48,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:49,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:50,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:50,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:51,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:52,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:53,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:54,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:55,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:55,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:44:56,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:57,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:58,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:44:59,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:00,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:00,070 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:00,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:01,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:01,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:02,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:03,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:04,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:05,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:06,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:06,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:07,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:08,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:09,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:10,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:11,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:11,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:12,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:13,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:14,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:15,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:15,077 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:15,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:16,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:45:16,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:16,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:45:16,403 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:45:16,403 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:45:16,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:16,405 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:45:17,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:17,353 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:45:18,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:19,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:19,355 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:45:20,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:21,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:22,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:22,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:23,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:24,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:25,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:26,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:27,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:28,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:28,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:29,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:30,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:30,074 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:30,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:31,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:32,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:33,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:34,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:34,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:35,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:36,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:37,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:38,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:39,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:39,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:40,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:41,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:42,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:43,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:44,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:44,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:45,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:45,089 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:45,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:46,150 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:45:46,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:47,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:47,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:45:47,791 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:45:47,791 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:45:47,793 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:45:48,377 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:45:48,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,378 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:45:49,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:50,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:51,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:52,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:53,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:54,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:55,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:55,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:56,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:57,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:58,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:59,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:00,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:00,080 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:00,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:01,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:01,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:02,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:03,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:04,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:05,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:06,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:06,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:07,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:08,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:09,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:10,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:11,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:12,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:12,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:13,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:14,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:15,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:15,083 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:15,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:16,155 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:46:16,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:17,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:18,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:18,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:19,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:46:19,399 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:46:19,400 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:46:19,401 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:46:19,415 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:46:19,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:20,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:21,417 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:46:21,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:22,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:23,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:23,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:24,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:25,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:26,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:27,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:28,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:28,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:29,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:30,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:30,084 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:30,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:31,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:32,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:33,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:34,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:34,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:35,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:36,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:37,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:38,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:39,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:39,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:40,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:41,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:42,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:43,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:44,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:44,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:45,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:45,085 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:45,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:46,158 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:46:46,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:47,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:48,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:49,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:46:49,284 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:46:49,285 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:46:49,286 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:46:49,451 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:46:49,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:50,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:50,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:51,453 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:46:51,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:52,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:53,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:54,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:55,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:55,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:56,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:57,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:58,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:59,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:00,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:00,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:00,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:01,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:01,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:02,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:03,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:04,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:05,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:06,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:06,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:07,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:08,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:09,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:10,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:11,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:12,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:12,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:13,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:14,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:15,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:15,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:15,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:16,165 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:47:16,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:17,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:18,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:18,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:19,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:47:19,577 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:47:19,577 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:19,579 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:19,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:20,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:47:20,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:21,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:47:21,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:22,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:23,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:23,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:24,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:25,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:26,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:27,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:28,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:29,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:29,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:30,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:30,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:30,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:31,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:32,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:33,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:34,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:35,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:35,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:36,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:37,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:38,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:39,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:40,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:41,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:41,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:42,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:43,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:44,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:45,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:45,097 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:45,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:46,170 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:47:46,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:47,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:48,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:48,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:49,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:50,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:47:50,768 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:47:50,768 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:50,769 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:51,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:51,527 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:47:52,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:52,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:53,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:53,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:47:54,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:55,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:56,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:57,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:59,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:00,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:00,092 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:00,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:01,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:02,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:03,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:04,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:04,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:05,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:06,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:07,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:08,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:09,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:09,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:10,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:11,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:12,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:13,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:14,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:15,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:15,092 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:15,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:15,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:16,183 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:48:16,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:17,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:18,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:19,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:20,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:20,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:21,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:21,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:48:21,535 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:48:21,535 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:21,537 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:21,557 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:48:22,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:23,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:23,559 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:48:24,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:25,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:25,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:26,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:27,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:28,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:29,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:30,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:30,092 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:30,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:31,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:31,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:32,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:33,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:34,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:35,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:37,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:38,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:39,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:40,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:41,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:42,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:42,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:43,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:44,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:45,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:45,111 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:45,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:46,174 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:48:46,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:47,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:48,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:48,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:49,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:50,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:51,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:48:51,041 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:48:51,042 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:51,044 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:51,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:51,587 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:48:52,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:53,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:53,588 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:48:54,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:54,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:55,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:56,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:57,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:58,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:59,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:59,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:00,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:00,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:00,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:01,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:02,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:03,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:04,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:05,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:05,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:06,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:07,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:08,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:09,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:10,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:10,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:11,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:12,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:13,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:14,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:15,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:15,104 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:15,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:15,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:16,178 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:49:16,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:17,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:18,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:19,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:49:19,351 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:49:19,351 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:19,352 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:19,613 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:49:19,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:20,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:20,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:21,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:21,615 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:49:22,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:23,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:24,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:25,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:26,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:26,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:27,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:28,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:29,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:30,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:30,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:30,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:31,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:32,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:32,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:33,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:34,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:35,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:36,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:37,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:37,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:38,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:39,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:40,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:41,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:42,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:43,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:43,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:44,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:45,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:45,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:45,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:46,191 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:49:46,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:47,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:48,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:49,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:49,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:50,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:51,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:51,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:49:51,986 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:49:51,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:51,988 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:52,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:52,640 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:49:53,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:53,641 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:49:54,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:55,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:55,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:56,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:57,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:58,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:59,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:00,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:00,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:00,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:00,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:01,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:02,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:03,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:04,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:05,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:05,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:06,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:07,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:08,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:09,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:10,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:10,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:11,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:12,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:13,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:14,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:15,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:15,115 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:15,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:15,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:16,188 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:50:16,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:17,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:18,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:19,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:20,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:20,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:20,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:20,657 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:50:20,657 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:20,662 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:20,669 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:50:21,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:22,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:23,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:23,672 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:50:24,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:25,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:26,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:26,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:27,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:28,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:29,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:30,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:30,107 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:30,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:31,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:32,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:32,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:33,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:34,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:35,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:36,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:37,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:37,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:38,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:39,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:40,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:41,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:42,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:43,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:43,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:44,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:45,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:45,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:45,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:46,199 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:50:46,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:47,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:48,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:49,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:49,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:50,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:51,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:51,071 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:50:51,071 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:51,074 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:51,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:51,694 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:50:52,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:53,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:53,695 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:50:54,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:55,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:55,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:56,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:57,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:58,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:59,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:00,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:00,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:00,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:00,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:01,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:02,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:03,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:04,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:05,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:05,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:06,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:07,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:08,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:09,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:10,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:11,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:11,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:12,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:13,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:14,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:15,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:15,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:15,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:16,196 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:51:16,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:17,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:17,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:18,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:19,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:20,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:21,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:22,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:22,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:23,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:24,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:25,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:51:25,303 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:51:25,303 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:51:25,307 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:25,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:25,732 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:51:26,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:27,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:27,733 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:51:27,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:28,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:29,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:30,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:30,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:30,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:31,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:32,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:33,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:33,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:34,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:35,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:36,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:37,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:38,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:38,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:39,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:40,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:41,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:42,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:43,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:43,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:44,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:45,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:45,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:45,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:46,200 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:51:46,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:47,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:48,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:49,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:49,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:50,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:51,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:52,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:53,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:54,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:54,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:55,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:56,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:57,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:58,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:51:58,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:51:58,678 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:51:58,684 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:58,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:51:59,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:59,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:59,761 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:52:00,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:00,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:00,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:01,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:02,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:03,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:04,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:05,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:05,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:06,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:07,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:08,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:09,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:10,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:10,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:11,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:12,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:13,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:14,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:15,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:15,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:15,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:15,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:16,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:16,204 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:52:17,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:18,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:19,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:20,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:21,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:21,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:22,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:23,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:24,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:25,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:26,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:26,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:27,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:28,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:28,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:52:28,738 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:52:28,738 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:52:28,741 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:52:28,787 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:52:29,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:29,788 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:52:30,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:30,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:30,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:31,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:32,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:32,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:33,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:34,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:35,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:36,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:37,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:38,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:38,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:39,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:40,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:41,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:42,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:43,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:43,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:44,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:45,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:45,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:45,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:46,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:46,208 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:52:47,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:48,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:49,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:49,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:50,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:51,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:52,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:53,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:54,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:54,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:55,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:56,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:56,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:52:56,505 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:52:56,506 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:52:56,508 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:52:56,814 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:52:57,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:57,816 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:52:58,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:59,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:00,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:00,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:00,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:00,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:01,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:02,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:03,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:04,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:05,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:05,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:06,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:07,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:08,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:09,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:10,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:10,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:11,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:12,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:13,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:14,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:15,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:15,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:15,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:16,212 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:53:16,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:16,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:17,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:18,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:19,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:20,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:21,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:21,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:22,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:23,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:24,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:25,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:26,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:26,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:27,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:28,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:29,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:53:29,136 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:53:29,136 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:29,138 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:29,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:29,855 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:53:30,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:30,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:30,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:31,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:31,857 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:53:32,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:32,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:33,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:34,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:35,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:36,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:37,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:38,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:38,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:39,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:40,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:41,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:42,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:43,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:44,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:44,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:45,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:45,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:45,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:46,216 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:53:46,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:47,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:48,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:49,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:50,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:50,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:51,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:52,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:53,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:54,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:55,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:55,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:56,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:57,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:58,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:53:58,482 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:53:58,482 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:58,483 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:58,885 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:53:58,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:59,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:59,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:54:00,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:00,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:00,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:01,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:01,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:02,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:03,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:04,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:05,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:06,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:07,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:07,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:08,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:09,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:10,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:11,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:12,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:12,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:13,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:14,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:15,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:15,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:15,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:16,220 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:54:16,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:17,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:18,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:18,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:19,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:20,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:21,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:22,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:23,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:23,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:24,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:25,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:26,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:27,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:28,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:28,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:29,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:54:29,459 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:54:29,459 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:54:29,461 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:54:29,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:29,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:54:30,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:30,110 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:30,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:31,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:31,914 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:54:32,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:33,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:33,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:34,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:35,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:36,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:37,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:38,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:39,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:39,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:40,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:41,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:42,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:43,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:44,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:44,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:45,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:45,111 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:45,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:46,223 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:54:46,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:47,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:48,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:49,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:50,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:50,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:51,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:52,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:53,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:54,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:55,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:55,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:56,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:57,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:58,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:59,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:00,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:00,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:00,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:00,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:01,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:55:01,151 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:55:01,151 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:01,152 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:01,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:01,948 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:55:02,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:03,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:03,949 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:55:04,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:05,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:05,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:06,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:07,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:08,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:09,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:10,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:11,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:11,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:12,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:14,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:14,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:15,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:15,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:15,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:16,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:55:16,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:17,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:17,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:18,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:19,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:20,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:21,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:22,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:22,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:23,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:24,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:25,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:26,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:27,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:27,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:28,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:29,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:30,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:30,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:30,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:31,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:32,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:55:32,141 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:55:32,142 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:32,143 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:32,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:32,980 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:55:33,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:33,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:33,981 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:55:34,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:35,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:36,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:37,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:38,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:38,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:39,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:40,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:41,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:42,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:43,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:43,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:44,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:45,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:45,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:45,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:46,231 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:55:46,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:47,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:48,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:49,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:49,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:50,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:51,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:52,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:53,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:54,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:54,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:55,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:56,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:57,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:58,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:59,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:55:59,414 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:55:59,414 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:59,415 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:59,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:00,012 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:56:00,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:00,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:00,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:00,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:01,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:02,014 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:56:02,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:03,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:04,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:05,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:05,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:06,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:07,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:08,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:09,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:10,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:11,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:11,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:13,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:14,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:15,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:15,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:15,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:16,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:16,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:56:17,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:17,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:18,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:19,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:20,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:21,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:22,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:22,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:23,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:24,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:25,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:26,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:27,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:27,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:28,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:29,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:30,126 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:30,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:30,571 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:56:30,571 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:30,572 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:31,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:31,041 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:56:32,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:32,042 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:56:32,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:33,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:34,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:35,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:36,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:37,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:38,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:38,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:39,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:40,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:41,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:42,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:43,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:44,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:44,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:45,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:45,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:45,126 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:46,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:46,238 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:56:47,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:48,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:49,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:50,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:50,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:51,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:52,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:53,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:54,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:55,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:55,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:56,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:57,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:58,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:59,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:00,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:00,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:57:00,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:57:00,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:00,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:57:00,803 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:57:00,803 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:57:00,805 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:57:01,080 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:57:01,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:02,081 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:57:02,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:03,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:04,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:05,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:06,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:06,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:07,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:08,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:09,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:10,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:11,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:12,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:12,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:13,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:14,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:15,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:57:15,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:57:15,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:16,242 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:57:16,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:17,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:18,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:18,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:19,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:20,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:21,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:22,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:23,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:23,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:24,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:25,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:26,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:27,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:28,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:28,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:29,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:30,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:57:30,073 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:57:30,073 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:57:30,075 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:57:30,109 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:57:30,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:30,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:57:30,494 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:57:31,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:32,111 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:57:32,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:33,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:34,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:34,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:35,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:36,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:37,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:38,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:39,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:40,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:40,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:41,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:42,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:43,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:44,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:45,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:57:45,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:57:45,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:46,246 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:57:46,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:46,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:47,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:48,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:49,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:50,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:51,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:51,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:52,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:53,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:54,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:55,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:55,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:57:55,752 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:57:55,752 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:57:55,754 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:57:56,150 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:57:56,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:56,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:57:57,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:58,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:57:58,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:57:59,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:00,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:58:00,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:58:00,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:01,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:02,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:02,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:03,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:04,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:05,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:06,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:07,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:08,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:08,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:09,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:10,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:11,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:12,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:13,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:13,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:14,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:15,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:58:15,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:58:15,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:16,251 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:58:16,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:17,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:18,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:19,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:19,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:20,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:21,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:22,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:23,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:24,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:24,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:25,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:26,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:58:26,344 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:58:26,345 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:58:26,346 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:58:27,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:58:27,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:28,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:58:28,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:29,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:30,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:58:30,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:58:30,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:30,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:31,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:32,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:33,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:34,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:35,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:35,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:36,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:37,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:38,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:39,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:40,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:41,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:41,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:42,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:43,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:44,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:45,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:58:45,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:58:45,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:46,256 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:58:46,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:47,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:47,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:48,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:49,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:50,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:51,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:52,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:52,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:53,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:54,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:55,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:56,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:57,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:57,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:58:58,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:58:58,158 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:58:58,158 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:58:58,159 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:58:58,219 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:58:58,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:58:59,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:00,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:59:00,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:59:00,226 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:59:00,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:01,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:02,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:03,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:03,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:04,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:05,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:06,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:07,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:08,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:09,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:09,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:10,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:11,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:12,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:13,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:14,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:14,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:15,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:59:15,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:59:15,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:16,259 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:59:16,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:17,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:18,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:19,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:20,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:20,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:21,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:22,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:23,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:24,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:25,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:25,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:26,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:27,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:59:27,426 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 13:59:27,426 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:59:27,427 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:59:28,248 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 13:59:28,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:29,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:30,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:59:30,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:59:30,249 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 13:59:30,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:31,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:31,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:32,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:33,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:34,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:35,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:36,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:36,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:37,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:38,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:39,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:40,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:41,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:41,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:42,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:43,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:44,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:45,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:59:45,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:59:45,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:46,264 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 13:59:46,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:47,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:47,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:48,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:49,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:50,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:51,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:52,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:52,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:53,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:54,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:55,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:56,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:57,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:57,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:59:58,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:59:59,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:00,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:00:00,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:00:00,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:01,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:00:01,340 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:00:01,341 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:00:01,342 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:00:01,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:02,286 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:00:02,287 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:00:02,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:03,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:03,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:04,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:05,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:06,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:07,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:08,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:08,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:09,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:10,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:11,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:12,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:13,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:14,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:14,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:15,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:00:15,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:00:15,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:16,268 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:00:16,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:17,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:18,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:19,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:20,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:20,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:21,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:22,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:23,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:24,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:25,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:25,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:26,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:27,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:28,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:29,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:30,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:00:30,221 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:00:30,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:30,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:00:30,554 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:00:30,555 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:00:30,556 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:00:30,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:31,318 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:00:31,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:32,318 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:00:32,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:33,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:34,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:35,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:36,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:36,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:37,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:38,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:39,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:40,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:41,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:42,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:42,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:43,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:44,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:45,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:00:45,293 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:00:45,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:46,272 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:00:46,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:47,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:48,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:48,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:49,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:50,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:51,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:52,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:53,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:53,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:54,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:55,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:56,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:57,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:58,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:00:58,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:00:59,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:00,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:01:00,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:01:00,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:01,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:02,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:03,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:01:03,009 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:01:03,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:01:03,011 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:01:03,351 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:01:03,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:04,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:04,353 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:01:04,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:05,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:06,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:07,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:08,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:09,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:09,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:10,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:11,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:12,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:13,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:14,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:14,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:15,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:01:15,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:01:15,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:16,276 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:01:16,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:17,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:18,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:19,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:20,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:20,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:21,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:22,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:23,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:24,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:25,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:25,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:26,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:27,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:28,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:29,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:30,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:01:30,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:01:30,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:30,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:01:30,570 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:01:30,571 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:01:30,572 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:01:30,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:31,379 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:01:31,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:32,380 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:01:32,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:33,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:34,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:35,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:36,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:36,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:37,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:38,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:39,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:40,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:41,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:42,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:42,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:43,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:44,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:45,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:01:45,292 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:01:45,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:46,280 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:01:46,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:47,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:48,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:48,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:49,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:50,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:51,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:52,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:53,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:53,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:54,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:55,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:56,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:57,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:58,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:01:58,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:01:59,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:00,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:02:00,304 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:02:00,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:01,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:02,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:02,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:02:02,818 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:02:02,818 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:02:02,819 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:02:03,406 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:02:03,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:04,407 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:02:04,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:04,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:05,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:06,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:07,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:08,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:09,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:09,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:10,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:11,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:12,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:13,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:14,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:15,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:02:15,295 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:02:15,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:15,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:16,284 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:02:16,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:17,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:18,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:19,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:20,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:20,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:21,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:22,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:23,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:24,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:25,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:26,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:26,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:27,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:28,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:29,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:30,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:02:30,302 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:02:30,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:31,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:32,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:32,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:33,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:34,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:02:34,219 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:02:34,219 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:02:34,220 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:02:34,433 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:02:34,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:35,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:36,436 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:02:36,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:37,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:38,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:38,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:39,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:40,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:41,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:42,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:43,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:44,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:44,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:45,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:02:45,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:02:45,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:46,288 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:02:46,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:47,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:48,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:49,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:49,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:50,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:51,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:52,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:53,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:54,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:55,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:02:55,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:56,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:57,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:58,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:02:59,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:00,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:00,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:03:00,302 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:03:00,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:01,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:02,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:03,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:03:03,565 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:03:03,566 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:03:03,567 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:03:03,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:04,462 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:03:04,462 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:03:04,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:05,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:05,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:06,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:07,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:08,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:09,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:10,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:11,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:11,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:12,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:13,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:14,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:15,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:03:15,303 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:03:15,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:16,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:16,292 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:03:16,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:17,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:18,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:19,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:20,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:21,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:22,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:22,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:23,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:24,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:25,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:26,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:27,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:28,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:28,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:29,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:30,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:03:30,310 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:03:30,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:31,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:32,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:33,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:33,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:03:33,685 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:03:33,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:03:33,687 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:03:34,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:34,491 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:03:34,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:03:34,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:35,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:36,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:37,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:38,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:39,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:39,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:40,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:41,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:42,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:43,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:44,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:45,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:45,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:03:45,310 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:03:45,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:46,298 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:03:46,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:47,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:48,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:49,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:50,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:50,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:51,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:52,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:53,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:54,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:55,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:56,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:03:56,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:57,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:58,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:03:59,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:00,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:04:00,315 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:04:00,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:01,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:01,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:02,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:03,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:04:03,808 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:04:03,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:04:03,810 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:04:04,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:04,524 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:04:04,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:05,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:06,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:06,526 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:04:06,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:07,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:08,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:09,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:10,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:11,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:11,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:12,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:13,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:14,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:15,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:04:15,313 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:04:15,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:16,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:16,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:04:16,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:17,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:18,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:19,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:20,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:21,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:22,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:22,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:23,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:24,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:25,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:26,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:27,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:27,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:28,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:29,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:30,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:04:30,315 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:04:30,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:31,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:04:31,207 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:04:31,208 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:04:31,209 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:04:31,567 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:04:31,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:32,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:32,568 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:04:32,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:33,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:34,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:35,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:36,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:37,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:38,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:38,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:39,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:40,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:41,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:42,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:44,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:44,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:45,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:45,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:04:45,315 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:04:46,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:46,305 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:04:47,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:48,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:49,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:49,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:50,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:51,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:52,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:53,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:54,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:55,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:04:55,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:56,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:57,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:58,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:04:59,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:00,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:00,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:00,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:05:00,327 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:05:01,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:01,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:05:01,748 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:05:01,749 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:05:01,750 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:05:02,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:02,593 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:05:02,593 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:05:03,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:04,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:05,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:05,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:06,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:07,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:08,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:09,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:10,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:11,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:11,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:12,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:13,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:14,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:15,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:15,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:05:15,317 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:05:16,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:16,309 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:05:16,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:17,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:18,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:19,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:20,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:21,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:22,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:22,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:23,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:24,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:25,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:26,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:27,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:27,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:28,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:29,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:30,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:30,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:05:30,317 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:05:31,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:31,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:05:31,790 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:05:31,790 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:05:31,792 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:05:32,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:32,626 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:05:32,627 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:05:32,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:33,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:34,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:35,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:36,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:37,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:38,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:38,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:39,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:40,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:41,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:42,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:43,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:44,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:44,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:45,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:45,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:05:45,498 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:05:46,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:46,310 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:05:47,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:48,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:49,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:49,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:50,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:51,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:52,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:53,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:54,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:55,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:05:55,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:56,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:57,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:58,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:05:59,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:00,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:00,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:00,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:06:00,392 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:06:01,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:01,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:06:01,352 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:06:01,353 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:06:01,355 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:06:01,659 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:06:02,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:02,660 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:06:03,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:04,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:05,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:05,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:06,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:07,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:08,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:09,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:10,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:11,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:11,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:12,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:13,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:14,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:15,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:15,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:06:15,392 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:06:16,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:16,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:16,313 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:06:17,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:18,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:19,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:20,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:21,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:21,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:22,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:23,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:24,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:25,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:26,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:26,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:27,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:28,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:29,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:30,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:30,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:06:30,398 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:06:30,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:06:30,839 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:06:30,840 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:06:30,841 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:06:31,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:31,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:31,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:06:32,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:32,691 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:06:33,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:34,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:35,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:36,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:37,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:37,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:38,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:39,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:40,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:41,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:42,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:43,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:43,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:44,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:45,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:45,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:06:45,395 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:06:46,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:46,317 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:06:47,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:48,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:48,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:49,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:50,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:51,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:52,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:53,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:54,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:54,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:55,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:56,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:57,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:58,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:06:59,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:06:59,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:00,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:00,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:07:00,402 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:07:01,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:01,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:07:01,653 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:07:01,654 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:07:01,655 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:07:01,718 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:07:02,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:02,719 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:07:03,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:04,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:04,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:05,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:06,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:07,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:08,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:09,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:10,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:10,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:11,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:12,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:13,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:14,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:15,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:15,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:07:15,401 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:07:15,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:16,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:16,322 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:07:17,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:18,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:19,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:20,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:21,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:21,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:22,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:23,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:24,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:25,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:26,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:27,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:27,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:28,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:29,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:30,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:30,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:07:30,409 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:07:31,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:32,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:32,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:33,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:07:33,098 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:07:33,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:07:33,100 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:07:33,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:33,753 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:07:34,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:34,754 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:07:35,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:36,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:37,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:38,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:38,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:39,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:40,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:41,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:42,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:43,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:44,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:44,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:45,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:07:45,506 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:07:45,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:46,328 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:07:46,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:47,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:48,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:49,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:49,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:50,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:51,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:52,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:53,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:54,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:55,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:07:55,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:56,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:57,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:58,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:07:59,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:00,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:00,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:08:00,420 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:08:00,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:01,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:02,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:03,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:08:03,036 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:08:03,036 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:08:03,038 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:08:03,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:03,791 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:08:04,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:04,792 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:08:05,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:06,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:06,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:07,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:08,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:09,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:10,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:11,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:11,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:12,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:13,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:14,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:15,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:08:15,421 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:08:15,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:16,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:16,330 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:08:16,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:17,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:18,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:19,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:20,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:21,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:22,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:22,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:23,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:24,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:25,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:26,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:27,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:27,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:28,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:29,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:30,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:08:30,426 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:08:30,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:31,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:32,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:08:32,280 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:08:32,280 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:08:32,282 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:08:32,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:32,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:32,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:08:32,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:08:33,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:34,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:35,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:36,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:37,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:38,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:38,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:39,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:40,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:41,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:42,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:43,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:44,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:44,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:45,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:08:45,427 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:08:45,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:46,334 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:08:46,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:47,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:48,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:49,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:49,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:50,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:51,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:52,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:53,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:54,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:55,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:08:55,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:56,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:57,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:58,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:08:59,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:00,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:00,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:09:00,430 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:09:00,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:01,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:01,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:09:01,932 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:09:01,932 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:09:01,934 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:09:02,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:02,853 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:09:02,854 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:09:03,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:04,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:05,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:05,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:06,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:07,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:08,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:09,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:10,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:11,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:11,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:12,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:13,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:14,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:15,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:09:15,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:09:15,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:16,338 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:09:16,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:16,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:17,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:18,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:19,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:20,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:21,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:22,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:22,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:23,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:24,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:25,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:26,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:27,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:27,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:28,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:29,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:30,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:09:30,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:09:30,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:09:30,554 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:09:30,555 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:09:30,556 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:09:30,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:30,878 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:09:31,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:31,879 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:09:32,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:33,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:33,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:34,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:35,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:36,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:37,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:38,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:39,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:39,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:40,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:41,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:42,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:43,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:44,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:45,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:45,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:09:45,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:09:45,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:46,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:09:46,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:47,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:48,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:49,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:50,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:50,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:51,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:52,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:53,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:54,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:55,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:56,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:09:56,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:57,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:58,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:09:59,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:00,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:10:00,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:10:00,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:01,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:01,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:02,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:03,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:04,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:10:04,107 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:10:04,108 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:10:04,109 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:10:04,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:04,907 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:10:05,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:05,907 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:10:06,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:07,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:07,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:08,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:09,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:10,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:11,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:12,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:12,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:13,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:14,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:15,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:10:15,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:10:15,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:16,346 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:10:16,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:17,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:17,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:18,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:19,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:20,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:21,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:22,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:23,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:23,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:24,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:25,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:26,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:27,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:28,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:28,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:29,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:30,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:10:30,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:10:30,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:31,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:32,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:10:32,767 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:10:32,768 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:10:32,769 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:10:32,941 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:10:33,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:33,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:33,942 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:10:34,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:35,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:36,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:37,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:38,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:39,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:39,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:40,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:41,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:42,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:43,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:44,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:45,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:45,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:45,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:10:45,434 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:10:46,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:46,350 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:10:47,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:48,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:49,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:50,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:50,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:51,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:52,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:53,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:54,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:55,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:56,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:56,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:10:57,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:58,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:10:59,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:00,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:00,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:11:00,435 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:11:01,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:01,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:02,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:03,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:03,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:11:03,755 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:11:03,756 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:11:03,757 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:11:03,981 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:11:04,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:05,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:05,983 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:11:06,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:07,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:07,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:08,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:09,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:10,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:11,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:12,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:13,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:13,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:14,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:15,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:15,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:11:15,435 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:11:16,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:16,354 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:11:17,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:18,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:18,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:19,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:20,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:21,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:22,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:23,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:24,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:24,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:25,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:26,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:27,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:28,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:29,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:29,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:30,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:30,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:11:30,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:11:31,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:32,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:33,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:33,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:11:33,708 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:11:33,709 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:11:33,710 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:11:34,009 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:11:34,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:34,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:35,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:36,010 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:11:36,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:37,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:38,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:39,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:40,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:40,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:41,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:42,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:43,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:44,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:45,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:45,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:45,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:11:45,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:11:46,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:46,358 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:11:47,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:48,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:49,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:50,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:50,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:51,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:52,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:53,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:54,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:55,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:56,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:56,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:11:57,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:58,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:11:59,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:00,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:00,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:12:00,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:12:01,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:01,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:02,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:03,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:04,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:05,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:05,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:12:05,370 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:12:05,371 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:12:05,372 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:12:06,043 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:12:06,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:07,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:07,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:08,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:12:08,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:09,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:10,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:11,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:12,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:13,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:13,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:14,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:15,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:15,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:12:15,442 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:12:16,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:16,363 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:12:17,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:18,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:18,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:19,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:20,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:21,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:22,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:23,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:24,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:24,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:25,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:26,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:27,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:28,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:29,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:29,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:30,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:30,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:12:30,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:12:31,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:32,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:33,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:34,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:34,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:35,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:35,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:12:35,290 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:12:35,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:12:35,292 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:12:36,072 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:12:36,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:37,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:38,074 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:12:38,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:39,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:40,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:40,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:41,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:42,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:43,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:44,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:45,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:45,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:12:45,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:12:45,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:46,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:46,366 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:12:47,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:48,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:49,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:50,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:51,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:51,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:52,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:53,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:54,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:55,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:56,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:56,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:12:57,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:58,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:12:59,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:00,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:00,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:13:00,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:13:01,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:01,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:02,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:03,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:04,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:05,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:05,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:13:05,608 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:13:05,609 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:13:05,611 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:13:06,104 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:13:06,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:07,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:07,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:08,105 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:13:08,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:09,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:10,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:11,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:12,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:13,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:13,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:14,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:15,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:15,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:13:15,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:13:16,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:16,370 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:13:17,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:18,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:19,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:19,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:20,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:21,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:22,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:23,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:24,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:24,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:25,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:26,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:27,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:28,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:29,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:30,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:30,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:30,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:13:30,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:13:31,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:32,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:33,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:34,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:35,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:35,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:36,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:37,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:37,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:13:37,332 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:13:37,332 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:13:37,333 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:13:38,136 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:13:38,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:39,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:40,138 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:13:40,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:41,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:41,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:42,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:43,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:44,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:45,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:45,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:13:45,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:13:46,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:46,374 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:13:46,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:47,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:48,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:49,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:50,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:51,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:51,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:52,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:53,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:54,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:55,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:56,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:57,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:57,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:13:58,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:13:59,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:00,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:00,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:14:00,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:14:01,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:02,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:02,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:03,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:04,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:05,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:06,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:07,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:07,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:08,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:09,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:09,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:14:09,268 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:14:09,269 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:14:09,270 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:14:10,174 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:14:10,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:11,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:12,175 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:14:12,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:13,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:13,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:14,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:15,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:15,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:14:15,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:14:16,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:16,378 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:14:17,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:18,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:18,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:19,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:20,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:21,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:22,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:23,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:24,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:24,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:25,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:26,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:27,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:28,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:29,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:29,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:30,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:30,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:14:30,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:14:31,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:32,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:33,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:34,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:34,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:35,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:36,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:37,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:37,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:14:37,898 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:14:37,898 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:14:37,900 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:14:38,201 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:14:38,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:39,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:39,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:40,203 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:14:40,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:41,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:42,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:43,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:44,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:45,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:45,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:45,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:14:45,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:14:46,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:46,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:14:47,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:48,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:49,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:50,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:50,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:51,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:52,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:53,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:54,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:55,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:56,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:56,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:14:57,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:58,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:14:59,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:00,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:00,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:15:00,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:15:01,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:02,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:02,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:03,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:04,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:05,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:06,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:07,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:07,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:08,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:09,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:10,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:10,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:15:10,740 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:15:10,740 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:15:10,742 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:15:11,232 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:15:11,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:12,233 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:15:12,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:13,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:13,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:14,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:15,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:15,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:15:15,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:15:16,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:16,387 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:15:17,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:18,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:18,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:19,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:20,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:21,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:22,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:23,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:24,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:24,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:25,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:26,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:27,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:28,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:29,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:29,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:30,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:15:30,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:15:30,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:31,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:32,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:33,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:34,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:35,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:35,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:36,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:37,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:38,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:39,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:40,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:40,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:41,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:42,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:43,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:43,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:15:43,731 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:15:43,731 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:15:43,733 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:15:44,266 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:15:44,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:45,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:15:45,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:15:45,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:45,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:46,268 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:15:46,391 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:15:46,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:47,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:48,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:49,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:50,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:51,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:51,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:52,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:53,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:54,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:55,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:56,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:15:56,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:57,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:58,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:15:59,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:00,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:16:00,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:16:00,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:01,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:02,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:02,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:03,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:04,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:05,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:06,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:07,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:07,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:08,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:09,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:10,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:11,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:12,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:13,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:13,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:14,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:14,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:16:14,782 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:16:14,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:16:14,784 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:16:15,295 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:16:15,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:16:15,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:16:15,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:16,296 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:16:16,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:16:16,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:17,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:18,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:19,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:19,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:20,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:21,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:22,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:23,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:24,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:24,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:25,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:26,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:27,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:28,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:29,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:30,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:30,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:16:30,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:16:30,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:31,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:32,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:33,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:34,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:35,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:36,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:36,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:37,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:38,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:39,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:40,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:41,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:41,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:42,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:43,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:44,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:45,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:16:45,456 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:16:45,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:46,402 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:16:46,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:46,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:46,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:16:46,831 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:16:46,831 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:16:46,833 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:16:47,327 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:16:47,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:48,327 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:16:48,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:49,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:50,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:51,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:52,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:52,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:53,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:54,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:55,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:56,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:57,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:58,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:16:58,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:16:59,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:00,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:17:00,457 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:17:00,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:01,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:02,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:03,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:04,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:04,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:05,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:06,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:07,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:08,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:09,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:09,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:10,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:11,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:12,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:13,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:14,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:14,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:15,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:17:15,459 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:17:15,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:16,404 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:17:16,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:17,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:18,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:19,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:17:19,439 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:17:19,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:17:19,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:19,441 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:17:19,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:20,365 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:17:20,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:21,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:22,367 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:17:22,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:23,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:24,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:25,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:25,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:26,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:27,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:28,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:29,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:30,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:17:30,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:17:30,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:30,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:31,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:32,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:33,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:34,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:35,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:36,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:36,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:37,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:38,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:39,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:40,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:41,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:42,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:42,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:43,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:44,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:45,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:17:45,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:17:45,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:46,402 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:17:46,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:47,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:47,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:48,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:17:48,483 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:17:48,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:17:48,485 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:17:48,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:49,400 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:17:49,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:50,401 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:17:50,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:51,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:52,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:53,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:53,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:54,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:55,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:56,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:57,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:58,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:17:59,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:17:59,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:00,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:18:00,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:18:00,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:01,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:02,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:03,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:04,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:04,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:05,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:06,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:07,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:08,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:09,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:09,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:10,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:11,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:12,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:13,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:14,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:15,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:15,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:18:15,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:18:15,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:16,405 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:18:16,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:17,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:18,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:19,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:20,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:20,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:21,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:22,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:23,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:18:23,320 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:18:23,320 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:18:23,325 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:18:23,432 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:18:23,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:24,433 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:18:24,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:25,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:26,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:26,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:27,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:28,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:29,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:30,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:18:30,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:18:30,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:31,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:31,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:32,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:33,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:34,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:35,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:36,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:36,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:37,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:38,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:39,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:40,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:41,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:42,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:42,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:43,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:44,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:45,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:18:45,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:18:45,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:46,409 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:18:46,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:47,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:48,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:48,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:49,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:50,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:51,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:52,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:53,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:18:53,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:18:53,793 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:18:53,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:53,798 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:18:54,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:54,463 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:18:54,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:55,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:56,465 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:18:56,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:57,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:58,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:18:59,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:18:59,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:00,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:19:00,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:19:00,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:01,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:02,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:03,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:04,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:05,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:05,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:06,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:07,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:08,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:09,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:10,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:10,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:11,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:12,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:13,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:14,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:15,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:19:15,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:19:15,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:15,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:16,413 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:19:16,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:17,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:18,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:19,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:20,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:21,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:21,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:22,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:23,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:19:23,691 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:19:23,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:19:23,693 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:19:24,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:24,494 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:19:25,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:26,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:26,496 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:19:27,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:27,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:28,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:29,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:30,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:30,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:19:30,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:19:31,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:32,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:32,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:33,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:34,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:35,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:36,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:37,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:38,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:38,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:39,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:40,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:41,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:42,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:43,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:43,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:44,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:45,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:45,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:19:45,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:19:46,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:46,417 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:19:47,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:48,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:49,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:49,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:50,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:51,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:52,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:53,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:54,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:54,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:19:54,665 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:19:54,665 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:19:54,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:19:54,667 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:19:55,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:55,520 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:19:56,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:56,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:19:57,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:58,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:19:59,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:00,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:00,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:00,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:20:00,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:20:01,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:02,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:03,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:04,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:05,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:06,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:06,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:07,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:08,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:09,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:10,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:11,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:11,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:12,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:13,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:14,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:15,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:15,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:20:15,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:20:16,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:16,422 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:20:17,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:17,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:18,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:19,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:20,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:21,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:22,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:22,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:23,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:24,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:24,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:20:24,845 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:20:24,845 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:20:24,846 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:20:25,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:25,545 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:20:26,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:26,546 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:20:27,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:28,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:28,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:29,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:30,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:30,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:20:30,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:20:31,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:32,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:33,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:34,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:34,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:35,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:36,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:37,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:38,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:39,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:40,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:40,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:41,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:42,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:43,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:44,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:45,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:45,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:20:45,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:20:45,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:46,426 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:20:46,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:47,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:48,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:49,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:50,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:51,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:51,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:52,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:53,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:54,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:55,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:56,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:20:56,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:57,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:58,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:20:58,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:20:58,561 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:20:58,561 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:20:58,566 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:20:58,578 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:20:59,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:00,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:00,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:21:00,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:21:00,580 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:21:01,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:01,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:02,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:03,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:04,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:05,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:06,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:07,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:07,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:08,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:09,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:10,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:11,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:12,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:13,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:13,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:14,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:15,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:15,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:21:15,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:21:16,430 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:21:16,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:17,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:18,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:19,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:19,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:20,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:21,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:22,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:23,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:24,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:24,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:25,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:26,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:27,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:28,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:29,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:29,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:29,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:21:29,540 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:21:29,541 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:21:29,543 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:21:29,610 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:21:30,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:30,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:21:30,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:21:30,611 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:21:31,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:32,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:33,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:34,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:35,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:35,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:36,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:37,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:38,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:39,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:40,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:40,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:41,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:42,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:43,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:44,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:45,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:45,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:21:45,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:21:45,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:46,434 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:21:46,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:47,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:48,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:49,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:50,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:51,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:51,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:52,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:53,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:54,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:55,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:56,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:21:56,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:57,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:58,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:21:59,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:00,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:00,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:22:00,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:22:01,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:01,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:22:01,723 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:22:01,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:22:01,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:01,727 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:22:02,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:02,642 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:22:02,642 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:22:03,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:04,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:05,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:06,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:07,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:07,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:08,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:09,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:10,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:11,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:12,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:13,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:13,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:14,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:15,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:22:15,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:22:15,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:16,438 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:22:16,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:17,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:18,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:18,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:19,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:20,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:21,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:22,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:23,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:24,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:24,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:25,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:26,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:27,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:28,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:29,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:29,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:30,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:22:30,468 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:22:30,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:30,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:22:30,627 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:22:30,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:22:30,628 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:22:30,673 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:22:31,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:32,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:32,675 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:22:33,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:34,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:35,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:35,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:36,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:37,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:38,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:39,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:40,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:41,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:41,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:42,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:43,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:44,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:45,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:22:45,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:22:45,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:46,442 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:22:46,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:46,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:47,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:48,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:49,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:50,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:51,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:51,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:52,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:53,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:54,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:55,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:56,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:22:56,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:57,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:58,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:22:59,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:00,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:23:00,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:23:00,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:01,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:02,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:02,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:03,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:23:03,138 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:23:03,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:23:03,140 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:23:03,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:03,705 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:23:04,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:04,706 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:23:05,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:06,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:07,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:07,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:08,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:09,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:10,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:11,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:12,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:13,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:13,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:14,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:15,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:23:15,474 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:23:15,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:16,447 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:23:16,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:17,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:18,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:18,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:19,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:20,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:21,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:22,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:23,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:23,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:24,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:25,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:26,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:27,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:28,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:28,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:29,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:30,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:23:30,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:23:30,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:31,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:32,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:33,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:33,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:34,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:23:34,168 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:23:34,169 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:23:34,171 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:23:34,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:34,738 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:23:35,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:36,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:36,739 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:23:37,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:38,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:39,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:39,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:40,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:41,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:42,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:43,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:44,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:45,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:45,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:23:45,474 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:23:45,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:46,450 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:23:46,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:47,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:48,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:49,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:50,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:51,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:51,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:52,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:53,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:54,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:55,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:56,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:23:56,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:57,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:58,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:23:59,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:00,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:24:00,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:24:00,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:01,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:01,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:24:01,962 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:24:01,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:24:01,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:01,964 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:24:02,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:02,764 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:24:02,764 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:24:03,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:04,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:05,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:06,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:07,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:07,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:08,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:09,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:10,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:11,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:12,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:13,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:13,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:14,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:15,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:24:15,479 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:24:15,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:16,454 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:24:16,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:17,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:18,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:19,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:19,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:20,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:21,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:22,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:23,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:24,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:25,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:25,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:26,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:27,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:28,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:29,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:30,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:30,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:24:30,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:24:30,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:31,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:31,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:24:31,794 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:24:31,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:24:31,795 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:24:32,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:32,790 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:24:32,791 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:24:33,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:34,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:35,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:35,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:36,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:37,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:38,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:39,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:40,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:41,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:41,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:42,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:43,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:44,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:45,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:24:45,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:24:45,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:46,458 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:24:46,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:47,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:47,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:48,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:49,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:50,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:51,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:52,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:52,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:53,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:54,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:55,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:56,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:57,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:24:57,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:58,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:24:59,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:00,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:25:00,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:25:00,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:01,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:02,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:03,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:25:03,220 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:25:03,221 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:25:03,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:03,222 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:25:03,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:03,819 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:25:04,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:04,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:25:05,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:06,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:07,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:08,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:08,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:09,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:10,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:11,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:12,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:13,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:13,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:14,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:15,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:25:15,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:25:15,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:16,462 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:25:16,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:17,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:18,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:18,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:19,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:20,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:21,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:22,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:23,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:24,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:24,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:25,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:26,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:27,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:28,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:29,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:29,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:30,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:25:30,483 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:25:30,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:31,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:32,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:33,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:34,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:35,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:25:35,124 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:25:35,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:25:35,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:35,126 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:25:35,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:35,849 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:25:36,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:36,850 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:25:37,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:38,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:39,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:40,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:40,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:41,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:42,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:43,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:44,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:45,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:25:45,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:25:45,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:45,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:46,467 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:25:46,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:47,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:48,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:49,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:50,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:51,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:51,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:52,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:53,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:54,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:55,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:56,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:25:56,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:57,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:58,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:25:59,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:00,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:26:00,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:26:00,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:01,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:02,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:02,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:03,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:04,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:05,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:06,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:07,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:07,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:08,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:26:08,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:26:08,228 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:26:08,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:26:08,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:08,875 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:26:09,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:10,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:10,877 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:26:11,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:12,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:13,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:13,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:14,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:15,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:26:15,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:26:15,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:16,470 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:26:16,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:17,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:18,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:19,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:19,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:20,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:21,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:22,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:23,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:24,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:24,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:25,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:26,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:27,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:28,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:29,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:29,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:30,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:26:30,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:26:30,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:31,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:32,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:33,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:34,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:35,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:35,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:36,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:37,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:38,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:39,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:40,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:40,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:41,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:42,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:43,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:26:43,324 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:26:43,325 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:26:43,327 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:26:43,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:43,909 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:26:44,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:44,910 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:26:45,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:26:45,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:26:45,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:45,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:46,474 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:26:46,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:47,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:48,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:49,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:50,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:51,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:51,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:52,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:53,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:54,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:55,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:56,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:26:56,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:57,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:58,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:26:59,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:00,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:27:00,484 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:27:00,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:01,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:02,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:02,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:03,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:04,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:05,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:06,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:07,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:08,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:08,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:09,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:10,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:11,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:12,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:13,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:14,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:27:14,094 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:27:14,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:27:14,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:14,098 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:27:14,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:14,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:27:14,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:27:15,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:27:15,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:27:15,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:16,477 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:27:16,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:17,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:18,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:19,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:19,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:20,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:21,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:22,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:23,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:24,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:25,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:25,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:26,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:27,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:28,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:29,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:30,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:27:30,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:27:30,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:30,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:31,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:32,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:33,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:34,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:35,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:36,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:36,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:37,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:38,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:39,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:40,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:41,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:41,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:42,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:43,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:44,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:45,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:27:45,492 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:27:45,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:46,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:27:46,443 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:27:46,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:27:46,444 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:27:46,481 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:27:46,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:46,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:46,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:27:47,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:48,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:48,977 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:27:49,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:50,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:51,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:52,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:52,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:53,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:54,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:55,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:56,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:58,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:58,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:27:58,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:27:59,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:00,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:28:00,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:28:00,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:01,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:02,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:03,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:03,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:04,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:05,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:06,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:07,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:08,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:09,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:10,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:10,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:11,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:12,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:13,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:14,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:15,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:28:15,447 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:28:15,447 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:28:15,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:15,449 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:28:15,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:28:15,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:28:15,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:16,005 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:28:16,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:28:16,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:17,006 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:28:17,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:18,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:19,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:20,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:20,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:21,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:22,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:23,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:24,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:25,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:26,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:27,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:28,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:29,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:30,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:30,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:28:30,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:28:31,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:32,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:32,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:33,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:34,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:35,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:36,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:37,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:38,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:38,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:39,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:40,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:41,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:42,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:43,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:43,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:43,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:28:43,624 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:28:43,624 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:28:43,626 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:28:44,033 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:28:44,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:45,034 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:28:45,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:45,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:28:45,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:28:46,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:46,490 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:28:47,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:48,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:49,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:49,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:50,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:51,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:52,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:53,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:54,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:55,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:55,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:28:56,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:57,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:58,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:28:59,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:00,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:00,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:29:00,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:29:00,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:01,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:02,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:03,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:04,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:05,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:06,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:06,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:07,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:08,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:09,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:10,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:11,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:11,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:12,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:13,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:14,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:15,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:15,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:29:15,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:29:15,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:29:15,927 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:29:15,928 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:29:15,929 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:29:16,064 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:29:16,494 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:29:16,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:17,065 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:29:17,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:17,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:18,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:19,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:20,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:21,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:22,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:22,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:23,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:24,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:25,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:26,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:27,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:28,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:28,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:29,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:30,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:30,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:29:30,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:29:31,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:32,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:33,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:33,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:34,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:35,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:36,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:37,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:38,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:38,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:39,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:40,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:41,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:42,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:43,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:43,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:44,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:45,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:45,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:29:45,598 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:29:46,496 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:29:46,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:47,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:48,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:48,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:29:48,595 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:29:48,595 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:29:48,596 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:29:48,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:49,103 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:29:49,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:50,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:51,105 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:29:51,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:52,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:53,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:54,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:54,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:29:55,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:56,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:57,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:58,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:59,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:29:59,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:00,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:00,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:30:00,599 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:30:01,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:02,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:03,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:04,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:04,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:05,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:06,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:07,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:08,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:09,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:10,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:10,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:11,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:12,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:13,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:14,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:15,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:15,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:30:15,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:30:15,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:16,504 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:30:16,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:17,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:18,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:19,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:30:19,101 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:30:19,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:30:19,103 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:30:19,135 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:30:19,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:20,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:21,137 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:30:21,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:21,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:22,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:23,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:24,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:25,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:26,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:26,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:27,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:28,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:29,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:30,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:30,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:30:30,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:30:31,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:32,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:32,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:33,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:34,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:35,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:36,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:37,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:37,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:38,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:39,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:40,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:41,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:42,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:43,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:43,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:44,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:45,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:45,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:30:45,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:30:46,508 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:30:46,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:47,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:48,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:48,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:49,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:50,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:51,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:52,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:53,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:30:53,116 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:30:53,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:30:53,118 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:30:53,170 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:30:53,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:54,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:30:54,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:55,171 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:30:55,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:56,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:57,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:58,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:59,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:30:59,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:00,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:00,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:31:00,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:31:01,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:02,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:03,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:04,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:05,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:05,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:06,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:07,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:08,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:09,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:10,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:10,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:11,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:12,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:13,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:14,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:15,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:15,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:31:15,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:31:15,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:16,517 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:31:16,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:17,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:18,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:19,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:20,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:21,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:21,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:22,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:31:22,507 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:31:22,508 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:31:22,509 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:31:22,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:23,203 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:31:23,204 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:31:23,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:24,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:25,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:26,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:26,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:27,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:28,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:29,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:30,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:31:30,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:31:30,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:31,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:32,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:32,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:33,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:34,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:35,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:36,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:37,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:37,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:38,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:39,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:40,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:41,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:42,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:43,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:43,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:44,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:45,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:31:45,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:31:45,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:46,518 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:31:46,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:47,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:48,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:48,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:49,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:50,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:51,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:52,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:53,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:31:53,150 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:31:53,150 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:31:53,152 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:31:53,234 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:31:53,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:54,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:54,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:55,236 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:31:55,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:56,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:57,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:58,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:31:59,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:31:59,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:00,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:32:00,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:32:00,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:01,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:02,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:03,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:04,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:04,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:05,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:06,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:07,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:08,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:09,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:09,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:10,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:11,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:12,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:13,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:14,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:15,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:32:15,606 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:32:15,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:16,527 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:32:16,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:17,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:18,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:19,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:19,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:20,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:21,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:21,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:32:21,894 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:32:21,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:32:21,896 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:32:22,264 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:32:22,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:23,265 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:32:23,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:24,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:24,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:25,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:26,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:27,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:28,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:29,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:30,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:32:30,607 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:32:30,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:30,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:31,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:32,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:33,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:34,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:35,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:36,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:36,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:37,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:38,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:39,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:40,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:41,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:42,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:42,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:43,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:44,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:45,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:32:45,611 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:32:45,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:46,537 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:32:46,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:47,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:48,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:48,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:49,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:32:49,299 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:32:49,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:32:49,301 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:32:49,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:50,296 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:32:50,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:51,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:32:51,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:52,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:53,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:53,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:54,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:55,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:56,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:57,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:58,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:32:59,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:32:59,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:00,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:33:00,608 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:33:00,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:01,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:02,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:03,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:04,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:04,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:05,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:06,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:07,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:08,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:09,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:09,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:10,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:11,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:12,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:13,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:14,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:15,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:33:15,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:33:15,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:15,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:16,533 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:33:16,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:17,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:18,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:19,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:20,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:21,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:33:21,418 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:33:21,418 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:33:21,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:21,423 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:33:21,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:22,327 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:33:22,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:23,328 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:33:23,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:24,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:25,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:26,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:26,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:27,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:28,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:29,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:30,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:33:30,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:33:30,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:31,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:32,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:32,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:33,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:34,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:35,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:36,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:37,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:38,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:38,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:39,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:40,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:41,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:42,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:43,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:43,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:44,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:45,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:33:45,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:33:45,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:46,536 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:33:46,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:47,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:48,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:49,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:33:49,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:33:49,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:33:49,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:49,228 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:33:49,360 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:33:49,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:50,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:51,361 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:33:51,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:52,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:53,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:54,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:33:54,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:55,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:56,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:57,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:58,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:33:59,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:00,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:34:00,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:34:00,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:00,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:01,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:02,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:03,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:04,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:05,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:05,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:06,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:07,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:08,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:09,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:10,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:10,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:11,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:12,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:13,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:14,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:15,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:34:15,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:34:15,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:16,540 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:34:16,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:16,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:17,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:18,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:19,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:20,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:21,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:21,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:22,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:34:22,517 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:34:22,518 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:34:22,520 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:34:22,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:23,394 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:34:23,395 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:34:23,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:24,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:25,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:26,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:26,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:27,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:28,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:29,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:30,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:34:30,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:34:30,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:31,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:32,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:32,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:33,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:34,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:35,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:36,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:37,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:37,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:38,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:39,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:40,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:41,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:42,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:43,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:43,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:44,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:45,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:34:45,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:34:45,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:46,544 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:34:46,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:47,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:48,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:48,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:49,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:50,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:51,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:34:51,869 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:34:51,869 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:34:51,870 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:34:52,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:52,423 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:34:53,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:53,424 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:34:53,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:34:54,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:55,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:56,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:57,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:58,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:59,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:34:59,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:00,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:00,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:35:00,616 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:35:01,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:02,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:03,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:04,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:04,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:05,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:06,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:07,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:08,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:09,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:09,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:10,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:11,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:12,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:13,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:14,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:15,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:15,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:35:15,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:35:15,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:16,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:16,548 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:35:17,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:18,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:19,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:20,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:20,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:21,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:21,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:35:21,618 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:35:21,619 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:35:21,620 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:35:22,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:22,455 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:35:23,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:23,456 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:35:24,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:25,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:26,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:26,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:27,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:28,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:29,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:30,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:30,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:35:30,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:35:31,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:31,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:32,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:33,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:34,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:35,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:36,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:37,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:37,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:38,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:39,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:40,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:41,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:42,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:43,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:43,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:44,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:45,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:45,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:35:45,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:35:46,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:46,552 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:35:47,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:48,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:49,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:49,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:49,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:35:49,115 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:35:49,115 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:35:49,117 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:35:49,484 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:35:50,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:51,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:51,485 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:35:52,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:53,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:54,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:35:54,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:55,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:56,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:57,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:58,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:35:59,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:00,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:00,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:00,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:36:00,624 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:36:01,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:02,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:03,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:04,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:05,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:06,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:06,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:07,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:08,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:09,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:10,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:11,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:11,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:12,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:13,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:14,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:15,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:15,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:36:15,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:36:16,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:16,561 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:36:16,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:17,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:18,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:19,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:20,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:20,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:36:20,566 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:36:20,567 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:36:20,569 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:36:21,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:21,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:36:21,522 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:36:21,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:22,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:23,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:24,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:25,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:26,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:27,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:27,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:28,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:29,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:30,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:30,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:36:30,624 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:36:31,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:32,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:33,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:33,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:34,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:35,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:36,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:37,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:38,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:38,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:39,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:40,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:41,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:42,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:43,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:44,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:44,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:45,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:45,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:36:45,629 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:36:46,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:46,572 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:36:47,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:48,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:49,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:49,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:50,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:50,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:36:50,663 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:36:50,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:36:50,665 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:36:51,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:51,548 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:36:52,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:52,549 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:36:53,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:54,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:55,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:36:55,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:56,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:57,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:58,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:36:59,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:00,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:00,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:37:00,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:37:00,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:01,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:02,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:03,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:04,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:05,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:06,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:06,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:07,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:08,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:09,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:10,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:11,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:12,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:12,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:13,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:14,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:15,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:15,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:37:15,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:37:16,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:16,566 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:37:17,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:17,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:18,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:19,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:19,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:37:19,741 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:37:19,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:37:19,747 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:37:20,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:20,581 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:37:21,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:22,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:22,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:22,583 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:37:23,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:24,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:25,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:26,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:27,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:27,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:28,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:29,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:30,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:30,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:37:30,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:37:31,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:32,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:33,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:33,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:34,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:35,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:36,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:37,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:38,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:38,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:39,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:40,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:41,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:42,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:43,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:43,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:44,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:45,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:45,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:37:45,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:37:46,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:46,576 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:37:47,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:48,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:48,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:49,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:50,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:51,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:52,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:37:52,344 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:37:52,344 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:37:52,346 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:37:52,612 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:37:52,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:53,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:53,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:54,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:54,613 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:37:55,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:56,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:57,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:58,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:37:59,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:37:59,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:00,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:00,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:38:00,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:38:01,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:02,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:03,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:04,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:05,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:05,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:06,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:07,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:08,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:09,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:10,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:10,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:11,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:12,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:13,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:14,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:15,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:15,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:15,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:38:15,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:38:16,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:16,574 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:38:17,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:18,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:19,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:20,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:20,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:21,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:22,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:23,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:38:23,008 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:38:23,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:38:23,011 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:38:23,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:23,655 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:38:24,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:24,656 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:38:25,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:26,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:26,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:27,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:28,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:29,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:30,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:30,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:38:30,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:38:31,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:31,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:32,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:33,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:34,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:35,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:36,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:37,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:37,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:38,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:39,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:40,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:41,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:42,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:43,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:43,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:44,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:45,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:45,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:38:45,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:38:46,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:46,578 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:38:47,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:48,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:49,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:49,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:50,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:51,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:52,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:53,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:54,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:54,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:38:54,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:38:54,236 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:38:54,241 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:38:54,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:54,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:38:55,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:56,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:56,691 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:38:57,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:58,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:38:59,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:38:59,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:00,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:39:00,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:39:00,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:01,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:02,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:03,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:04,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:05,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:05,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:06,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:07,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:08,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:09,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:10,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:10,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:11,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:12,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:13,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:14,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:15,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:39:15,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:39:15,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:15,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:16,581 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:39:16,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:17,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:18,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:19,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:20,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:21,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:21,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:22,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:23,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:23,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:39:23,875 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:39:23,876 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:39:23,877 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:39:24,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:24,715 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:39:25,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:26,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:26,717 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:39:26,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:27,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:28,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:29,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:30,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:39:30,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:39:30,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:31,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:32,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:32,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:33,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:34,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:35,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:36,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:37,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:38,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:38,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:39,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:40,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:41,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:42,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:43,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:43,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:44,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:45,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:39:45,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:39:45,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:46,585 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:39:46,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:47,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:48,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:49,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:49,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:50,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:51,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:52,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:53,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:54,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:39:54,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:55,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:39:55,360 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:39:55,360 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:39:55,362 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:39:55,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:39:55,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:56,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:57,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:58,750 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:39:58,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:39:59,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:00,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:00,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:40:00,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:40:00,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:01,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:02,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:03,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:04,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:05,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:05,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:06,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:07,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:08,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:09,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:10,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:11,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:11,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:12,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:13,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:14,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:15,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:40:15,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:40:15,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:16,601 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:40:16,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:16,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:17,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:18,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:19,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:20,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:21,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:40:21,823 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:40:21,824 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:40:21,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:21,826 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:40:22,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:22,774 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:40:23,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:24,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:24,776 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:40:25,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:26,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:27,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:27,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:28,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:29,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:30,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:30,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:40:30,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:40:31,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:32,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:33,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:33,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:34,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:35,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:36,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:37,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:38,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:38,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:39,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:40,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:41,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:42,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:43,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:44,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:44,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:45,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:45,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:40:45,645 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:40:46,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:46,594 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:40:47,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:48,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:49,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:49,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:50,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:51,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:51,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:40:51,367 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:40:51,367 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:40:51,369 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:40:51,805 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:40:52,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:52,806 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:40:53,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:54,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:55,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:40:55,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:56,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:57,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:58,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:40:59,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:00,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:00,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:41:00,631 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:41:00,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:01,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:02,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:03,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:04,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:05,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:06,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:06,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:07,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:08,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:09,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:10,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:11,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:11,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:12,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:13,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:14,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:15,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:15,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:41:15,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:41:16,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:16,597 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:41:16,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:17,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:18,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:19,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:20,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:21,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:22,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:22,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:23,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:24,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:24,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:41:24,786 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:41:24,787 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:41:24,789 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:41:24,840 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:41:25,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:26,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:26,842 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:41:27,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:28,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:28,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:29,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:30,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:30,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:41:30,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:41:31,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:32,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:33,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:34,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:34,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:35,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:36,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:37,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:38,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:39,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:40,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:40,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:41,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:42,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:43,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:44,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:45,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:45,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:45,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:41:45,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:41:46,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:46,601 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:41:47,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:48,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:49,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:50,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:50,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:51,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:52,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:53,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:54,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:55,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:41:55,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:56,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:56,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:41:56,718 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:41:56,718 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:41:56,720 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:41:56,890 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:41:58,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:41:58,891 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:41:59,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:00,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:00,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:42:00,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:42:00,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:01,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:02,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:03,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:04,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:05,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:06,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:06,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:07,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:08,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:09,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:10,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:11,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:12,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:12,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:13,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:14,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:15,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:15,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:42:15,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:42:16,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:16,606 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:42:17,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:17,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:18,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:19,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:20,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:21,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:22,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:23,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:23,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:24,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:25,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:26,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:27,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:27,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:42:27,799 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:42:27,799 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:42:27,805 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:42:27,917 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:42:28,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:28,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:28,919 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:42:29,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:30,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:30,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:42:30,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:42:31,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:32,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:33,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:34,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:34,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:35,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:36,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:37,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:38,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:39,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:40,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:40,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:41,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:42,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:43,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:44,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:45,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:45,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:45,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:42:45,632 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:42:46,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:46,610 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:42:47,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:48,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:49,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:50,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:51,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:51,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:52,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:53,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:54,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:55,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:56,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:42:56,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:57,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:58,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:42:59,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:00,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:00,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:43:00,634 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:43:00,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:43:00,951 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:43:00,952 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:43:00,954 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:43:01,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:01,948 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:43:01,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:02,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:02,949 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:43:03,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:04,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:05,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:06,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:07,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:07,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:08,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:09,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:10,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:11,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:12,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:12,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:13,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:14,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:15,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:15,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:43:15,633 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:43:16,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:16,614 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:43:17,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:17,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:18,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:19,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:20,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:21,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:22,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:23,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:23,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:24,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:25,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:26,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:27,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:28,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:28,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:29,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:30,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:30,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:43:30,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:43:31,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:43:31,177 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:43:31,178 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:43:31,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:43:31,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:31,981 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:43:32,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:32,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:43:33,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:34,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:34,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:35,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:36,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:37,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:38,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:39,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:40,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:40,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:41,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:42,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:43,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:44,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:45,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:45,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:45,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:43:45,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:43:46,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:46,619 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:43:47,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:48,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:49,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:50,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:51,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:51,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:52,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:53,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:54,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:55,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:56,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:43:56,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:57,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:58,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:43:59,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:00,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:00,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:44:00,632 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:44:00,633 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:44:00,634 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:44:01,011 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:44:01,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:44:01,053 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:44:01,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:02,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:02,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:03,013 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:44:03,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:04,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:05,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:06,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:07,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:07,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:08,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:09,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:10,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:11,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:12,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:13,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:13,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:14,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:15,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:15,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:44:15,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:44:16,622 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:44:16,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:17,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:18,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:19,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:19,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:20,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:21,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:22,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:23,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:24,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:24,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:25,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:26,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:44:26,165 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:44:26,165 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:44:26,167 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:44:26,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:27,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:44:27,046 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:44:27,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:28,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:29,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:30,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:30,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:44:30,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:44:30,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:31,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:32,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:33,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:34,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:35,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:36,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:36,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:37,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:38,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:39,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:41,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:42,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:42,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:43,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:44,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:45,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:45,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:44:45,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:44:46,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:46,627 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:44:47,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:47,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:48,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:49,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:50,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:51,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:52,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:53,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:53,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:54,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:55,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:56,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:57,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:58,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:59,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:44:59,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:44:59,391 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:44:59,392 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:44:59,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:44:59,393 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:45:00,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:00,076 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:45:00,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:45:00,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:45:01,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:01,078 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:45:02,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:03,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:04,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:04,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:05,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:06,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:07,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:08,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:09,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:10,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:10,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:11,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:12,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:13,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:14,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:15,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:15,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:15,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:45:15,787 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:45:16,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:16,630 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:45:17,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:18,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:19,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:20,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:21,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:21,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:22,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:23,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:24,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:25,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:26,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:26,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:27,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:28,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:29,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:30,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:30,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:45:30,096 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:45:30,096 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:45:30,097 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:45:30,112 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:45:30,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:45:30,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:45:31,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:31,116 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:45:31,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:32,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:33,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:34,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:35,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:36,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:37,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:37,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:38,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:39,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:40,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:41,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:42,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:43,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:43,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:44,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:45,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:45,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:45:45,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:45:46,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:46,634 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:45:47,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:48,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:48,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:49,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:50,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:51,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:52,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:53,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:54,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:54,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:45:55,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:56,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:57,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:58,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:59,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:45:59,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:00,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:00,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:46:00,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:46:01,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:02,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:03,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:04,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:04,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:46:04,247 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:46:04,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:46:04,248 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:46:05,151 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:46:05,151 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:46:05,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:05,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:06,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:07,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:08,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:09,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:10,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:10,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:11,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:12,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:13,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:14,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:15,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:15,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:46:15,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:46:15,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:16,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:16,638 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:46:17,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:18,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:19,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:20,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:21,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:21,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:22,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:23,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:24,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:25,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:26,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:27,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:27,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:28,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:29,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:30,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:30,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:46:30,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:46:31,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:32,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:32,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:33,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:34,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:35,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:36,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:36,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:46:36,602 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:46:36,602 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:46:36,604 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:46:37,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:37,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:46:38,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:38,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:39,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:39,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:46:40,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:41,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:42,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:43,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:44,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:44,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:45,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:45,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:46:45,791 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:46:46,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:46,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:46:47,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:48,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:49,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:49,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:50,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:51,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:52,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:53,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:54,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:55,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:55,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:46:56,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:57,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:58,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:46:59,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:00,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:00,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:00,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:47:00,805 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:47:01,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:02,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:03,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:04,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:05,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:05,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:06,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:06,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:47:06,651 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:47:06,651 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:47:06,653 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:47:07,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:07,218 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:47:08,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:09,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:09,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:47:10,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:10,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:11,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:12,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:13,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:14,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:15,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:15,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:47:15,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:47:15,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:16,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:16,646 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:47:17,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:18,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:19,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:20,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:21,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:21,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:22,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:23,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:24,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:25,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:26,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:26,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:27,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:28,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:29,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:30,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:30,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:47:30,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:47:31,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:31,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:32,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:33,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:34,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:35,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:36,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:37,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:37,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:38,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:38,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:47:38,945 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:47:38,945 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:47:38,946 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:47:39,253 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:47:39,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:40,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:41,255 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:47:41,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:42,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:42,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:43,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:44,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:45,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:45,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:47:45,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:47:46,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:46,651 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:47:47,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:48,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:48,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:49,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:50,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:51,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:52,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:53,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:53,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:54,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:55,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:56,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:57,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:58,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:47:58,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:47:59,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:00,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:00,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:48:00,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:48:01,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:02,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:03,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:04,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:04,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:05,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:06,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:07,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:08,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:09,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:09,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:10,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:48:10,170 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:48:10,170 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:48:10,172 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:48:10,282 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:48:10,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:11,283 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:48:11,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:12,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:13,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:14,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:15,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:15,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:15,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:48:15,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:48:16,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:16,654 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:48:17,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:18,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:19,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:20,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:21,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:21,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:22,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:23,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:24,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:25,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:26,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:27,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:27,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:28,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:29,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:30,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:30,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:48:30,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:48:31,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:32,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:32,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:33,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:34,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:35,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:36,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:37,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:38,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:38,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:39,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:40,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:41,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:42,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:48:42,228 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:48:42,229 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:48:42,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:48:42,316 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:48:42,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:43,317 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:48:43,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:43,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:44,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:45,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:45,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:48:45,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:48:46,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:46,658 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:48:47,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:48,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:49,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:49,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:50,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:51,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:52,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:53,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:54,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:48:54,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:55,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:56,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:57,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:58,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:48:59,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:00,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:00,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:00,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:49:00,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:49:01,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:02,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:03,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:04,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:05,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:06,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:06,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:07,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:08,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:09,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:10,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:11,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:11,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:12,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:13,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:49:13,089 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:49:13,089 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:49:13,090 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:49:13,348 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:49:13,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:14,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:15,350 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:49:15,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:15,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:49:15,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:49:16,661 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:49:16,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:16,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:17,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:18,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:19,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:20,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:21,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:22,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:22,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:23,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:24,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:25,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:26,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:27,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:28,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:28,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:29,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:30,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:30,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:49:30,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:49:31,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:32,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:33,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:34,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:34,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:35,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:36,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:37,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:38,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:39,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:39,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:40,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:41,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:42,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:43,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:49:43,134 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:49:43,134 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:49:43,135 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:49:43,378 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:49:43,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:44,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:44,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:45,381 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:49:45,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:45,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:49:45,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:49:46,665 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:49:46,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:47,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:48,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:49,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:50,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:50,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:51,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:52,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:53,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:54,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:55,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:56,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:49:56,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:57,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:58,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:49:59,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:00,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:00,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:50:00,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:50:01,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:01,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:02,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:03,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:04,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:05,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:06,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:07,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:07,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:08,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:09,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:10,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:11,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:12,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:12,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:13,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:14,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:15,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:50:15,636 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:50:15,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:50:15,638 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:50:16,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:16,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:50:16,056 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:50:16,414 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:50:16,670 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:50:16,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:17,414 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:50:17,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:17,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:18,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:19,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:20,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:21,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:22,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:23,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:23,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:24,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:25,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:26,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:27,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:28,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:28,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:29,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:30,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:31,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:50:31,052 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:50:31,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:32,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:33,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:34,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:34,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:35,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:36,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:37,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:38,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:39,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:39,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:40,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:41,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:42,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:43,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:50:43,026 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:50:43,026 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:50:43,028 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:50:43,443 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:50:43,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:44,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:45,445 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:50:45,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:45,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:46,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:50:46,053 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:50:46,674 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:50:46,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:47,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:48,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:49,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:50,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:51,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:51,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:52,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:53,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:54,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:55,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:56,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:57,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:50:57,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:58,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:50:59,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:00,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:01,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:51:01,053 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:51:01,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:02,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:03,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:03,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:04,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:05,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:06,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:07,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:08,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:08,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:09,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:10,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:11,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:12,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:13,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:13,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:14,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:15,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:51:15,663 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:51:15,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:51:15,665 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:51:16,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:16,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:51:16,079 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:51:16,476 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:51:16,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:51:17,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:17,477 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:51:18,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:19,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:19,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:20,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:21,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:22,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:23,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:24,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:24,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:25,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:26,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:27,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:28,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:29,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:30,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:30,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:31,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:51:31,075 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:51:31,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:32,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:33,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:34,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:35,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:35,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:36,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:37,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:38,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:39,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:40,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:41,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:41,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:42,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:43,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:44,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:45,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:46,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:51:46,079 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:51:46,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:46,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:51:46,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:47,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:48,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:48,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:51:48,630 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:51:48,630 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:51:48,632 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:51:49,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:49,519 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:51:49,520 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:51:50,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:51,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:52,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:52,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:53,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:54,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:55,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:56,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:57,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:58,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:51:58,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:51:59,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:00,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:01,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:52:01,077 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:52:01,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:02,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:03,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:03,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:04,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:05,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:06,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:07,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:08,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:09,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:09,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:10,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:11,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:12,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:13,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:14,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:14,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:15,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:16,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:52:16,078 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:52:16,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:16,687 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:52:17,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:17,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:52:17,811 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:52:17,811 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:52:17,814 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:52:18,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:18,546 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:52:19,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:19,547 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:52:19,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:20,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:21,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:22,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:23,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:24,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:25,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:25,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:26,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:27,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:28,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:29,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:30,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:31,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:52:31,079 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:52:31,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:31,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:32,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:33,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:34,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:35,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:36,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:36,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:37,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:38,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:39,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:40,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:41,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:42,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:42,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:43,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:44,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:45,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:46,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:52:46,091 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:52:46,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:46,696 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:52:47,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:47,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:48,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:48,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:52:48,369 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:52:48,370 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:52:48,373 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:52:48,573 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:52:49,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:49,575 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:52:50,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:51,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:52,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:53,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:53,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:54,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:55,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:56,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:57,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:58,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:52:58,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:52:59,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:00,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:01,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:53:01,082 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:53:01,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:02,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:03,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:04,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:04,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:05,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:06,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:07,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:08,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:09,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:10,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:10,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:11,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:12,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:13,589 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:53:15,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:16,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:16,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:53:21,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:22,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:23,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:53:23,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:53:26,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:31,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:32,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:32,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:37,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:39,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:40,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:53:40,331 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:53:40,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:41,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:42,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:46,702 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:53:47,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:48,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:50,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:53,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:57,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:57,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:53:57,529 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:53:58,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:53:58,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:53:59,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:03,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:08,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:11,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:11,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:11,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:54:11,983 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:54:12,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:13,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:13,935 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:54:14,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:14,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:15,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:16,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:16,701 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:54:17,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:18,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:19,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:20,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:20,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:21,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:22,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:23,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:24,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:25,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:26,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:26,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:26,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:54:26,983 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:54:27,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:28,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:29,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:30,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:31,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:32,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:32,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:33,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:34,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:35,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:36,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:37,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:38,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:38,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:39,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:40,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:41,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:41,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:54:41,983 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:54:42,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:43,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:44,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:44,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:45,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:46,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:46,705 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:54:47,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:48,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:49,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:49,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:50,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:51,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:52,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:53,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:54,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:55,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:54:55,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:56,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:56,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:54:56,983 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:54:57,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:58,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:54:59,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:00,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:01,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:01,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:02,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:03,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:04,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:05,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:06,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:11,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:16,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:16,709 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:55:21,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:26,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:26,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:55:26,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:55:26,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:27,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:28,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:29,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:30,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:31,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:31,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:32,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:33,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:34,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:35,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:36,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:36,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:37,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:38,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:39,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:40,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:41,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:55:41,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:55:41,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:41,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:42,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:43,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:44,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:45,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:46,715 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:55:46,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:47,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:47,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:48,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:49,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:50,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:51,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:52,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:52,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:53,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:54,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:55,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:56,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:55:56,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:55:56,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:57,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:57,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:55:58,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:55:59,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:00,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:01,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:02,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:03,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:03,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:04,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:05,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:06,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:07,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:08,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:08,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:09,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:10,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:11,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:56:11,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:56:11,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:12,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:13,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:13,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:14,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:15,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:16,717 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:56:16,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:17,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:18,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:19,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:19,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:20,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:21,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:22,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:23,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:24,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:24,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:25,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:26,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:56:26,784 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:56:26,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:27,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:28,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:29,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:29,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:30,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:31,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:32,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:33,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:34,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:35,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:35,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:36,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:37,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:38,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:39,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:40,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:41,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:56:41,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:56:41,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:41,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:42,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:43,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:44,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:45,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:46,722 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:56:46,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:47,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:47,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:48,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:49,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:50,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:51,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:52,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:52,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:53,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:54,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:55,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:56,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:56:56,784 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:56:56,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:57,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:58,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:56:58,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:56:59,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:00,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:01,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:02,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:03,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:04,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:04,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:05,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:06,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:07,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:08,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:09,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:10,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:10,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:11,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:57:11,783 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:57:11,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:12,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:13,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:14,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:15,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:15,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:16,726 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:57:16,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:17,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:18,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:19,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:20,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:21,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:21,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:22,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:23,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:57:26,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:31,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:36,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:41,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:46,730 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:57:47,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:52,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:57:57,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:02,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:02,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:58:02,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:02,857 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:58:03,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:04,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:05,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:06,085 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:58:06,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:07,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:07,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:08,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:09,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:10,088 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:58:10,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:11,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:12,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:13,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:13,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:14,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:15,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:16,734 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:58:16,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:17,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:17,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:58:17,855 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:58:18,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:18,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:19,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:20,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:21,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:22,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:23,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:24,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:24,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:25,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:26,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:27,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:28,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:29,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:30,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:30,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:31,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:32,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:32,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:58:32,870 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:58:33,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:34,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:35,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:36,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:36,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:37,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:58:37,409 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:58:37,410 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:58:37,411 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:58:37,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:38,122 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:58:38,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:39,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:40,123 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:58:40,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:41,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:41,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:42,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:43,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:44,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:45,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:46,739 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:58:46,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:47,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:47,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:58:47,857 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:58:47,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:48,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:49,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:50,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:51,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:52,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:52,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:53,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:54,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:55,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:56,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:57,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:58,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:58:58,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:58:59,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:00,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:01,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:02,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:59:02,861 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:59:02,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:03,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:04,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:04,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:05,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:06,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:59:06,448 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:59:06,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:59:06,449 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:59:06,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:07,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:59:07,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:08,153 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:59:08,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:09,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:09,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:10,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:11,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:12,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:13,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:14,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:14,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:15,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:16,742 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:59:16,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:17,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:59:17,861 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:59:17,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:18,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:19,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:19,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:20,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:21,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:22,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:23,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:24,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:25,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:25,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:26,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:27,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:28,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:29,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:30,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:30,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:31,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:32,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:59:32,875 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:59:32,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:33,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:34,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:35,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 14:59:35,799 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 14:59:35,800 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 14:59:35,802 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 14:59:36,184 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 14:59:36,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:36,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:37,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:38,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 14:59:38,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:39,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:40,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:41,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:41,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:42,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:43,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:44,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:45,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:46,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:46,746 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 14:59:47,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:47,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:47,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 14:59:47,864 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 14:59:48,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:49,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:50,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:51,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:52,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:52,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:53,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:54,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:55,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:56,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:57,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:58,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 14:59:58,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 14:59:59,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:00,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:01,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:02,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:02,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:00:02,868 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:00:03,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:04,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:04,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:05,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:05,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:00:05,864 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:00:05,864 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:00:05,865 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:00:06,214 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:00:06,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:07,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:08,216 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:00:08,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:09,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:09,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:10,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:11,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:12,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:13,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:14,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:15,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:15,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:16,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:16,749 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:00:17,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:17,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:00:17,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:00:18,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:19,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:20,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:20,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:21,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:22,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:23,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:24,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:25,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:26,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:26,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:27,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:28,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:29,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:30,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:31,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:32,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:32,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:00:32,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:00:32,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:33,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:34,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:35,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:36,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:37,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:38,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:38,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:39,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:40,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:41,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:00:41,026 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:00:41,026 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:00:41,028 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:00:41,248 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:00:41,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:42,249 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:00:42,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:43,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:44,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:44,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:45,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:46,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:46,753 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:00:47,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:47,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:00:47,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:00:48,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:49,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:50,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:50,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:51,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:52,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:53,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:54,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:55,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:55,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:00:56,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:57,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:58,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:00:59,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:00,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:00,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:01,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:02,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:02,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:01:02,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:01:03,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:04,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:05,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:06,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:06,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:07,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:08,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:09,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:10,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:11,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:11,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:01:11,476 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:01:11,476 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:01:11,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:11,478 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:01:12,276 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:01:12,277 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:01:12,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:13,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:14,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:15,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:16,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:16,758 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:01:16,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:17,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:17,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:01:17,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:01:18,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:19,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:20,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:21,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:21,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:22,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:23,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:24,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:25,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:26,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:27,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:27,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:28,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:29,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:30,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:31,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:32,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:32,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:01:32,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:01:32,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:33,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:34,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:35,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:36,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:37,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:38,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:38,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:39,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:40,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:41,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:42,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:01:42,303 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:01:42,304 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:01:42,305 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:01:42,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:43,299 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:01:43,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:44,300 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:01:44,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:44,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:45,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:46,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:46,761 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:01:47,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:47,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:01:47,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:01:48,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:49,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:49,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:50,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:51,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:52,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:53,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:54,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:55,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:55,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:01:56,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:57,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:58,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:01:59,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:00,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:00,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:01,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:02,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:02,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:02:02,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:02:03,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:04,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:05,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:06,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:06,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:07,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:08,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:09,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:10,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:11,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:11,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:12,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:02:12,018 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:02:12,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:02:12,019 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:02:12,323 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:02:12,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:13,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:14,325 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:02:14,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:15,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:16,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:16,765 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:02:17,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:17,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:17,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:02:17,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:02:18,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:19,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:20,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:21,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:22,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:22,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:23,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:24,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:25,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:26,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:27,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:27,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:28,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:29,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:30,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:31,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:32,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:32,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:02:32,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:02:33,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:33,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:34,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:35,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:36,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:37,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:38,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:38,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:39,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:40,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:41,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:42,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:02:42,182 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:02:42,182 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:02:42,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:02:42,351 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:02:42,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:43,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:44,353 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:02:44,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:44,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:45,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:46,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:46,769 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:02:47,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:47,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:02:47,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:02:48,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:49,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:49,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:50,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:51,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:52,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:53,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:54,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:55,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:55,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:02:56,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:57,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:58,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:02:59,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:00,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:01,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:01,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:02,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:02,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:03:02,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:03:03,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:04,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:05,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:06,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:06,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:07,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:08,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:09,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:10,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:11,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:12,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:03:12,032 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:03:12,033 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:03:12,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:12,034 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:03:12,380 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:03:12,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:13,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:14,383 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:03:14,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:15,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:16,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:16,774 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:03:17,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:17,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:17,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:03:17,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:03:18,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:19,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:20,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:21,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:22,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:22,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:23,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:24,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:25,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:26,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:27,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:27,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:28,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:29,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:30,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:31,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:32,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:32,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:03:32,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:03:33,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:33,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:34,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:35,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:36,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:37,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:38,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:38,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:39,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:40,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:41,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:42,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:43,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:03:43,469 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:03:43,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:03:43,471 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:03:43,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:44,408 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:03:44,408 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:03:44,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:44,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:45,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:46,778 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:03:46,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:47,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:03:47,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:03:47,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:48,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:49,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:50,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:50,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:51,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:52,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:53,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:54,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:55,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:55,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:03:56,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:57,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:58,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:03:59,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:00,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:00,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:01,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:02,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:04:02,867 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:04:02,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:03,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:04,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:05,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:05,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:06,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:07,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:08,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:09,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:10,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:11,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:11,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:12,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:13,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:14,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:15,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:16,784 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:04:16,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:17,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:04:17,223 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:04:17,223 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:04:17,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:17,224 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:04:17,438 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:04:17,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:04:17,867 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:04:17,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:18,439 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:04:18,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:19,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:20,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:21,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:22,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:22,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:23,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:24,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:25,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:26,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:27,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:27,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:28,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:29,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:30,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:31,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:32,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:04:32,867 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:04:32,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:33,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:33,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:34,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:35,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:36,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:37,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:38,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:38,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:39,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:40,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:41,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:42,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:43,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:43,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:44,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:45,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:46,788 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:04:46,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:47,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:04:47,843 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:04:47,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:04:47,844 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:04:48,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:48,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:04:48,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:04:48,468 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:04:48,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:49,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:49,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:50,471 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:04:50,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:51,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:52,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:53,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:54,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:55,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:04:55,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:56,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:57,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:58,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:04:59,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:00,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:01,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:01,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:02,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:05:02,995 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:05:03,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:03,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:04,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:05,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:06,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:06,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:07,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:08,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:09,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:11,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:11,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:11,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:12,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:14,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:15,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:16,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:16,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:16,787 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:05:16,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:05:16,842 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:05:16,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:05:16,844 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:05:17,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:17,495 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:05:18,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:05:18,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:05:18,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:18,496 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:05:19,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:20,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:21,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:22,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:22,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:23,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:24,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:25,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:26,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:27,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:27,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:28,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:29,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:30,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:31,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:32,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:32,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:33,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:05:33,001 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:05:33,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:34,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:35,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:36,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:37,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:38,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:38,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:39,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:40,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:41,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:42,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:43,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:43,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:44,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:45,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:05:45,041 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:05:45,041 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:05:45,042 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:05:45,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:45,529 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:05:46,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:46,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:05:46,791 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:05:47,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:48,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:05:48,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:05:48,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:49,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:49,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:50,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:51,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:52,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:53,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:54,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:54,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:55,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:56,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:57,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:58,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:05:59,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:05:59,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:00,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:01,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:02,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:03,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:06:03,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:06:03,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:04,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:05,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:05,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:06,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:07,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:08,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:09,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:10,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:10,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:11,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:12,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:13,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:14,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:15,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:15,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:06:15,684 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:06:15,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:06:15,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:15,688 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:06:16,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:16,558 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:06:16,559 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:06:16,796 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:06:17,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:18,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:06:18,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:06:18,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:19,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:20,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:21,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:21,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:22,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:23,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:24,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:25,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:26,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:27,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:27,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:28,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:29,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:30,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:31,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:32,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:32,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:33,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:06:33,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:06:33,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:34,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:35,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:36,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:37,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:37,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:38,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:39,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:40,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:41,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:42,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:42,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:43,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:44,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:45,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:46,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:46,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:06:46,798 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:06:46,801 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:06:46,801 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:06:46,802 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:06:47,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:47,588 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:06:47,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:48,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:06:48,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:06:48,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:48,589 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:06:49,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:50,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:51,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:52,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:53,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:53,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:54,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:55,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:56,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:57,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:58,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:06:58,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:06:59,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:00,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:01,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:02,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:03,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:07:03,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:07:03,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:04,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:04,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:05,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:06,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:07,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:08,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:09,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:09,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:10,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:11,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:12,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:12,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:07:12,892 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:07:12,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:07:12,894 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:07:13,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:13,626 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:07:14,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:14,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:14,627 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:07:15,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:16,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:16,801 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:07:17,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:18,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:07:18,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:07:18,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:19,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:20,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:20,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:21,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:22,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:23,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:24,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:25,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:25,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:26,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:27,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:28,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:29,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:30,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:30,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:31,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:32,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:33,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:07:33,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:07:33,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:34,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:35,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:35,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:36,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:37,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:38,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:39,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:40,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:40,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:40,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:07:40,606 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:07:40,607 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:07:40,609 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:07:40,652 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:07:41,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:42,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:42,653 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:07:43,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:44,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:45,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:46,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:46,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:46,805 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:07:47,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:48,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:07:48,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:07:48,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:49,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:50,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:51,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:52,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:52,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:53,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:54,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:55,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:56,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:57,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:07:57,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:58,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:07:59,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:00,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:01,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:02,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:03,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:08:03,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:08:03,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:03,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:04,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:05,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:06,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:07,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:08,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:08,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:09,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:10,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:11,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:08:11,270 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:08:11,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:08:11,272 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:08:11,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:11,686 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:08:12,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:12,687 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:08:13,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:14,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:14,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:15,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:16,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:16,810 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:08:17,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:18,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:08:18,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:08:18,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:19,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:20,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:20,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:21,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:22,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:23,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:24,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:25,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:26,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:26,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:27,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:28,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:29,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:30,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:31,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:31,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:32,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:33,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:08:33,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:08:33,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:34,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:35,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:36,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:36,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:37,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:38,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:39,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:40,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:41,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:41,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:42,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:08:42,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:08:42,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:08:42,088 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:08:42,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:42,713 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:08:42,713 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:08:43,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:44,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:45,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:46,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:46,811 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:08:46,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:47,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:48,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:08:48,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:08:48,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:49,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:50,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:51,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:52,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:52,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:53,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:54,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:55,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:56,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:57,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:08:57,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:58,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:08:59,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:00,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:01,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:02,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:02,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:03,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:09:03,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:09:03,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:04,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:05,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:06,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:07,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:08,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:08,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:09,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:10,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:11,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:12,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:13,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:13,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:14,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:15,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:09:15,593 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:09:15,594 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:09:15,595 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:09:15,752 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:09:16,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:16,753 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:09:16,815 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:09:17,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:18,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:09:18,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:09:18,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:19,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:19,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:20,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:21,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:22,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:23,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:24,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:25,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:25,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:26,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:27,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:28,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:29,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:30,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:30,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:31,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:32,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:33,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:09:33,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:09:33,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:34,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:35,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:36,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:36,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:37,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:38,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:39,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:40,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:41,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:41,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:42,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:43,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:44,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:45,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:45,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:09:45,442 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:09:45,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:09:45,444 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:09:45,780 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:09:46,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:46,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:46,781 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:09:46,820 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:09:47,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:48,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:09:48,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:09:48,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:49,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:50,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:51,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:52,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:52,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:53,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:54,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:55,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:56,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:57,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:58,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:09:58,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:09:59,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:00,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:01,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:02,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:03,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:10:03,006 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:10:03,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:04,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:04,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:05,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:06,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:07,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:08,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:09,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:09,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:10,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:11,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:12,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:13,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:14,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:14,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:15,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:16,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:16,821 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:10:17,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:17,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:10:17,164 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:10:17,165 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:10:17,170 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:10:17,813 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:10:18,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:10:18,006 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:10:18,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:18,815 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:10:19,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:20,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:20,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:21,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:22,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:23,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:24,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:25,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:25,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:26,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:27,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:28,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:29,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:30,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:31,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:31,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:32,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:33,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:10:33,006 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:10:33,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:34,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:35,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:36,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:36,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:37,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:38,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:39,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:40,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:41,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:42,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:42,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:43,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:44,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:45,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:46,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:46,830 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:10:47,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:47,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:48,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:10:48,013 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:10:48,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:49,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:49,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:10:49,544 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:10:49,545 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:10:49,547 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:10:49,847 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:10:50,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:50,848 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:10:51,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:52,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:53,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:53,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:10:54,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:55,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:56,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:57,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:58,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:59,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:10:59,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:00,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:01,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:02,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:03,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:11:03,008 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:11:03,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:04,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:05,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:05,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:06,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:07,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:08,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:09,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:10,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:10,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:11,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:12,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:13,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:14,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:15,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:15,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:16,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:16,830 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:11:17,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:18,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:11:18,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:11:18,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:19,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:20,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:20,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:20,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:11:20,704 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:11:20,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:11:20,707 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:11:20,882 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:11:21,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:22,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:22,884 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:11:23,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:24,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:25,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:26,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:26,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:27,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:28,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:29,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:30,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:31,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:32,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:32,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:33,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:11:33,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:11:33,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:34,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:35,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:36,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:37,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:37,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:38,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:39,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:40,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:41,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:42,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:42,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:43,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:44,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:45,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:46,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:46,844 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:11:47,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:47,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:48,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:11:48,014 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:11:48,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:49,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:50,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:51,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:52,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:52,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:11:52,441 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:11:52,441 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:11:52,445 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:11:52,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:11:53,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:53,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:54,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:54,915 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:11:55,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:56,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:57,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:58,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:11:58,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:11:59,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:00,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:01,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:02,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:03,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:12:03,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:12:03,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:04,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:04,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:05,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:06,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:07,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:08,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:09,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:09,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:10,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:11,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:12,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:13,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:14,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:15,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:15,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:16,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:16,839 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:12:17,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:18,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:12:18,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:12:18,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:19,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:20,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:20,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:21,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:22,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:23,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:24,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:25,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:25,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:12:25,585 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:12:25,586 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:12:25,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:25,589 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:12:25,945 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:12:26,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:26,946 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:12:27,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:28,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:29,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:30,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:31,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:31,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:32,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:33,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:12:33,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:12:33,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:34,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:35,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:36,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:37,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:37,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:38,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:39,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:40,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:41,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:42,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:42,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:43,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:44,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:45,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:46,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:46,844 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:12:47,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:47,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:48,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:12:48,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:12:48,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:49,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:50,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:51,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:52,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:53,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:53,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:54,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:54,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:12:54,903 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:12:54,904 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:12:54,905 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:12:54,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:12:55,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:56,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:56,983 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:12:57,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:58,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:12:58,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:12:59,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:00,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:01,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:02,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:03,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:13:03,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:13:03,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:04,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:04,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:05,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:06,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:07,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:08,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:09,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:09,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:10,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:11,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:12,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:13,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:14,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:15,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:15,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:16,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:16,847 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:13:17,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:18,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:13:18,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:13:18,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:19,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:20,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:20,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:21,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:22,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:23,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:24,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:25,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:13:25,170 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:13:25,170 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:13:25,172 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:13:25,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:26,008 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:13:26,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:26,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:27,017 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:13:27,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:28,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:29,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:30,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:31,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:31,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:32,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:33,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:13:33,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:13:33,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:34,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:35,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:36,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:37,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:37,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:38,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:39,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:40,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:41,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:42,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:43,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:43,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:44,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:45,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:46,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:46,852 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:13:47,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:48,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:13:48,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:13:48,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:49,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:49,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:50,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:51,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:52,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:53,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:54,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:54,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:55,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:56,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:57,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:13:57,626 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:13:57,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:13:57,631 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:13:58,051 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:13:58,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:58,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:13:59,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:13:59,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:00,053 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:14:00,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:01,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:02,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:03,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:14:03,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:14:03,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:04,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:05,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:05,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:06,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:07,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:08,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:09,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:10,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:10,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:11,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:12,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:13,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:14,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:15,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:15,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:16,856 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:14:16,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:17,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:18,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:14:18,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:14:18,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:19,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:20,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:21,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:21,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:22,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:23,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:24,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:25,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:26,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:26,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:27,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:28,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:14:28,784 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:14:28,785 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:14:28,790 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:14:29,085 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:14:29,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:29,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:30,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:31,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:31,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:32,087 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:14:32,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:33,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:14:33,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:14:33,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:34,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:35,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:36,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:37,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:37,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:38,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:39,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:40,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:41,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:42,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:43,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:43,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:44,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:45,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:46,860 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:14:46,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:47,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:48,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:14:48,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:14:48,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:49,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:49,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:50,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:51,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:52,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:53,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:54,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:14:54,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:55,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:56,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:57,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:58,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:14:58,763 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:14:58,763 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:14:58,765 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:14:59,113 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:14:59,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:14:59,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:00,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:01,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:02,115 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:15:02,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:03,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:15:03,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:15:03,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:04,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:05,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:05,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:06,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:07,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:08,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:09,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:10,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:11,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:11,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:12,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:13,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:14,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:15,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:16,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:16,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:16,865 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:15:17,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:18,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:15:18,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:15:18,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:19,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:20,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:21,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:22,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:22,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:23,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:24,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:25,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:26,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:27,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:27,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:28,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:29,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:15:29,201 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:15:29,202 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:15:29,203 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:15:29,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:30,145 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:15:30,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:31,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:32,147 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:15:32,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:33,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:15:33,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:15:33,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:33,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:34,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:35,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:36,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:37,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:38,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:38,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:39,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:40,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:41,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:42,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:43,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:43,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:44,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:45,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:46,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:46,868 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:15:47,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:48,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:15:48,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:15:48,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:49,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:49,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:50,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:51,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:52,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:53,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:54,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:54,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:15:55,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:56,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:57,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:58,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:15:59,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:15:59,186 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:15:59,186 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:15:59,188 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:15:59,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:00,169 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:16:00,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:00,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:01,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:02,170 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:16:02,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:03,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:16:03,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:16:03,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:04,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:05,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:05,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:06,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:07,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:08,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:09,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:10,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:10,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:11,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:12,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:13,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:14,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:15,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:15,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:16,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:16,872 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:16:17,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:18,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:16:18,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:16:18,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:19,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:20,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:21,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:21,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:22,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:23,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:24,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:25,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:16:25,659 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:16:25,660 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:16:25,662 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:16:26,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:26,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:16:26,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:27,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:28,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:28,204 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:16:29,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:30,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:31,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:32,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:32,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:33,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:16:33,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:16:33,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:34,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:35,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:36,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:37,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:38,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:38,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:39,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:40,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:41,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:42,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:43,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:44,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:44,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:45,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:46,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:46,877 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:16:47,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:48,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:16:48,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:16:48,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:49,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:50,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:50,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:51,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:52,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:53,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:54,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:55,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:56,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:56,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:16:56,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:16:56,545 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:16:56,545 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:16:56,547 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:16:57,233 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:16:57,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:58,234 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:16:58,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:16:59,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:00,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:01,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:01,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:02,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:03,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:17:03,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:17:03,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:04,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:05,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:06,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:07,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:07,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:08,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:09,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:10,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:11,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:12,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:12,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:13,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:14,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:15,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:16,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:16,880 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:17:17,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:17,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:18,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:17:18,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:17:18,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:19,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:20,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:21,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:22,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:23,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:23,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:24,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:25,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:26,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:17:26,192 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:17:26,192 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:17:26,194 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:17:26,269 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:17:27,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:28,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:28,270 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:17:28,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:29,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:30,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:31,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:32,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:33,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:17:33,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:17:33,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:34,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:34,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:35,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:36,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:37,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:38,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:39,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:39,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:40,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:41,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:42,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:43,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:44,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:45,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:45,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:46,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:46,885 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:17:47,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:48,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:17:48,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:17:48,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:49,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:50,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:51,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:51,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:52,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:53,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:54,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:55,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:56,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:56,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:17:57,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:57,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:17:57,646 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:17:57,646 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:17:57,648 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:17:58,311 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:17:58,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:17:59,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:00,313 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:18:00,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:01,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:02,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:02,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:03,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:18:03,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:18:03,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:04,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:05,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:06,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:07,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:08,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:08,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:09,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:10,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:11,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:12,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:13,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:13,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:14,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:15,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:16,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:16,889 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:18:17,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:18,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:18:18,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:18:18,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:19,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:19,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:20,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:21,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:22,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:23,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:24,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:25,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:25,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:18:25,404 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:18:25,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:18:25,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:25,406 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:18:26,338 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:18:26,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:27,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:28,340 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:18:28,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:29,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:30,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:30,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:31,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:32,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:33,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:18:33,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:18:33,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:34,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:35,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:36,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:36,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:37,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:38,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:39,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:40,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:41,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:41,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:42,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:43,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:44,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:45,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:46,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:46,893 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:18:46,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:47,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:48,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:18:48,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:18:48,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:49,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:50,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:51,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:52,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:52,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:53,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:54,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:55,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:56,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:18:56,132 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:18:56,133 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:18:56,134 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:18:56,368 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:18:57,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:58,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:18:58,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:18:58,370 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:18:59,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:00,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:01,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:02,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:03,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:19:03,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:19:03,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:03,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:04,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:05,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:06,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:07,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:08,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:08,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:09,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:10,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:11,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:12,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:13,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:14,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:14,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:15,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:16,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:16,897 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:19:17,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:18,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:19:18,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:19:18,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:19,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:20,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:20,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:21,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:22,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:23,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:24,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:25,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:25,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:26,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:27,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:28,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:29,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:30,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:30,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:19:30,050 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:19:30,050 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:19:30,052 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:19:30,413 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:19:31,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:31,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:32,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:32,416 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:19:33,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:19:33,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:19:33,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:34,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:35,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:36,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:36,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:37,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:38,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:39,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:40,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:41,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:42,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:42,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:43,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:44,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:45,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:46,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:46,901 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:19:47,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:47,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:48,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:19:48,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:19:48,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:49,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:50,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:51,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:52,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:53,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:53,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:54,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:55,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:56,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:57,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:58,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:19:58,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:19:59,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:00,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:00,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:20:00,286 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:20:00,286 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:20:00,288 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:20:00,452 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:20:01,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:02,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:02,453 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:20:03,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:20:03,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:20:03,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:04,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:04,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:05,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:06,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:07,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:08,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:09,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:10,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:10,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:11,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:12,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:13,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:14,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:15,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:15,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:16,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:16,905 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:20:17,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:18,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:20:18,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:20:18,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:19,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:20,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:21,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:21,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:22,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:23,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:24,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:25,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:26,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:26,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:27,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:28,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:29,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:30,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:30,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:20:30,275 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:20:30,275 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:20:30,276 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:20:30,481 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:20:31,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:32,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:32,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:32,483 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:20:33,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:20:33,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:20:33,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:34,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:35,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:36,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:37,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:37,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:38,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:39,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:40,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:41,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:42,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:43,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:43,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:44,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:45,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:46,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:46,909 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:20:47,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:48,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:20:48,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:20:48,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:49,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:49,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:50,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:51,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:52,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:53,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:54,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:55,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:55,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:20:56,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:57,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:58,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:20:59,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:00,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:00,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:01,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:01,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:21:01,450 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:21:01,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:21:01,457 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:21:01,515 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:21:02,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:02,516 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:21:03,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:21:03,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:21:03,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:04,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:05,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:06,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:06,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:07,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:08,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:09,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:10,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:11,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:11,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:12,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:13,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:14,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:15,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:16,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:16,912 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:21:16,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:17,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:18,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:21:18,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:21:18,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:19,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:20,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:21,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:22,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:22,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:23,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:24,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:25,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:26,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:27,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:27,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:28,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:29,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:30,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:31,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:32,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:32,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:21:32,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:21:32,486 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:21:32,487 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:21:32,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:32,541 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:21:33,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:21:33,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:21:33,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:34,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:34,543 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:21:35,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:36,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:37,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:38,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:38,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:39,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:40,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:41,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:42,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:43,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:43,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:44,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:45,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:46,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:46,916 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:21:47,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:48,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:21:48,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:21:48,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:48,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:49,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:50,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:51,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:52,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:53,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:54,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:54,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:21:55,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:56,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:57,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:58,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:59,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:21:59,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:00,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:01,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:02,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:03,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:22:03,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:22:03,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:03,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:22:03,776 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:22:03,777 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:22:03,778 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:22:04,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:04,572 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:22:04,572 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:22:04,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:05,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:06,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:07,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:08,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:09,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:10,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:10,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:11,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:12,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:13,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:14,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:15,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:16,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:16,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:16,920 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:22:17,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:18,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:22:18,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:22:18,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:19,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:20,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:21,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:21,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:22,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:23,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:24,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:25,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:26,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:27,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:27,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:28,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:29,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:30,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:31,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:32,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:32,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:33,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:22:33,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:22:33,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:33,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:22:33,870 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:22:33,871 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:22:33,872 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:22:34,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:34,602 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:22:35,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:36,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:36,603 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:22:37,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:37,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:38,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:39,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:40,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:41,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:42,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:43,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:43,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:44,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:45,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:46,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:46,924 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:22:47,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:48,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:22:48,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:22:48,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:48,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:49,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:50,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:51,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:52,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:53,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:54,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:54,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:22:55,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:56,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:57,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:58,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:59,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:22:59,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:00,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:01,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:02,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:03,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:23:03,025 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:23:03,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:04,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:04,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:04,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:23:04,902 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:23:04,902 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:23:04,904 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:23:05,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:05,635 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:23:06,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:06,636 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:23:07,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:08,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:09,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:10,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:10,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:11,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:12,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:13,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:14,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:15,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:16,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:16,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:16,929 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:23:17,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:18,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:23:18,024 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:23:18,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:19,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:20,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:21,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:21,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:22,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:23,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:24,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:25,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:26,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:27,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:27,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:28,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:29,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:30,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:31,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:32,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:32,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:33,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:23:33,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:23:33,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:34,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:35,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:35,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:23:35,533 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:23:35,533 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:23:35,535 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:23:35,659 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:23:36,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:36,660 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:23:37,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:37,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:38,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:39,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:40,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:41,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:42,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:43,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:43,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:44,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:45,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:46,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:46,932 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:23:47,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:48,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:23:48,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:23:48,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:48,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:49,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:50,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:51,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:52,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:53,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:53,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:23:54,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:55,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:56,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:57,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:58,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:59,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:23:59,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:00,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:01,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:02,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:03,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:24:03,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:24:03,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:04,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:04,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:05,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:06,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:07,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:08,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:08,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:24:08,847 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:24:08,847 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:24:08,849 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:24:09,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:09,694 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:24:09,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:10,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:10,695 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:24:11,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:12,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:13,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:14,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:15,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:15,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:16,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:16,936 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:24:17,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:18,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:24:18,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:24:18,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:19,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:20,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:21,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:21,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:22,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:23,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:24,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:25,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:26,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:26,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:27,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:28,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:29,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:30,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:31,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:32,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:32,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:33,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:24:33,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:24:33,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:34,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:35,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:36,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:37,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:37,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:38,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:39,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:40,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:24:40,407 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:24:40,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:24:40,408 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:24:40,727 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:24:40,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:41,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:42,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:42,729 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:24:43,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:43,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:44,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:45,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:46,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:46,940 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:24:47,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:48,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:24:48,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:24:48,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:48,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:49,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:50,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:51,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:52,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:53,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:54,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:54,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:55,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:56,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:57,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:58,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:24:59,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:24:59,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:00,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:01,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:02,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:03,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:25:03,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:25:03,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:04,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:05,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:05,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:06,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:07,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:08,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:09,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:10,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:10,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:11,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:12,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:12,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:25:12,919 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:25:12,920 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:25:12,921 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:25:13,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:13,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:25:14,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:14,761 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:25:15,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:16,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:16,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:16,944 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:25:17,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:18,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:25:18,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:25:18,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:19,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:20,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:21,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:22,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:22,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:23,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:24,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:25,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:26,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:27,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:27,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:28,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:29,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:30,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:31,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:32,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:32,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:33,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:25:33,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:25:33,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:34,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:35,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:36,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:37,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:37,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:38,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:39,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:40,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:41,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:42,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:42,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:42,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:25:42,787 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:25:42,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:25:42,789 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:25:42,790 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:25:43,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:44,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:44,792 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:25:45,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:46,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:46,947 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:25:47,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:47,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:48,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:25:48,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:25:48,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:49,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:50,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:51,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:52,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:53,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:53,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:54,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:55,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:56,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:57,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:58,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:25:58,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:25:59,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:00,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:01,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:02,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:03,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:26:03,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:26:03,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:04,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:04,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:05,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:06,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:07,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:08,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:09,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:09,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:10,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:11,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:12,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:12,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:26:12,987 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:26:12,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:26:12,989 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:26:13,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:13,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:26:14,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:14,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:26:15,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:15,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:16,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:16,951 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:26:17,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:18,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:26:18,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:26:18,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:19,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:20,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:21,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:21,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:22,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:23,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:24,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:25,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:26,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:27,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:27,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:28,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:29,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:30,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:31,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:32,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:32,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:33,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:26:33,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:26:33,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:34,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:35,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:36,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:37,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:37,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:38,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:39,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:40,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:41,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:42,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:42,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:42,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:26:42,804 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:26:42,804 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:26:42,805 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:26:42,850 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:26:43,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:44,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:44,851 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:26:45,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:46,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:46,955 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:26:47,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:47,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:48,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:26:48,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:26:48,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:49,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:50,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:51,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:52,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:53,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:53,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:54,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:55,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:56,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:57,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:58,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:26:58,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:26:59,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:00,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:01,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:02,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:03,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:27:03,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:27:03,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:04,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:04,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:05,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:06,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:07,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:08,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:09,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:09,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:10,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:11,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:12,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:13,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:14,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:14,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:27:14,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:27:14,694 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:27:14,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:14,695 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:27:14,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:27:15,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:16,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:16,888 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:27:16,959 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:27:17,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:18,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:27:18,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:27:18,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:19,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:20,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:20,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:21,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:22,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:23,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:24,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:25,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:26,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:26,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:27,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:28,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:29,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:30,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:31,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:31,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:32,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:33,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:27:33,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:27:33,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:34,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:35,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:36,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:36,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:37,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:38,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:39,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:40,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:41,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:41,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:42,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:43,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:44,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:45,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:46,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:46,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:46,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:27:46,921 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:27:46,921 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:27:46,922 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:27:46,923 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:27:46,963 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:27:47,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:48,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:27:48,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:27:48,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:48,924 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:27:49,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:50,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:51,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:52,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:52,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:53,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:54,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:55,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:56,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:57,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:27:57,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:58,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:27:59,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:00,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:01,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:02,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:02,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:03,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:28:03,029 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:28:03,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:04,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:05,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:06,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:07,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:08,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:08,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:09,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:10,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:11,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:12,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:13,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:13,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:14,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:15,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:28:15,353 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:28:15,354 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:28:15,355 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:28:15,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:15,955 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:28:16,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:16,956 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:28:16,966 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:28:17,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:18,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:28:18,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:28:18,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:18,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:19,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:20,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:21,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:22,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:23,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:23,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:24,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:25,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:26,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:27,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:28,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:29,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:29,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:30,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:31,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:32,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:33,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:28:33,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:28:33,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:34,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:34,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:35,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:36,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:37,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:38,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:39,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:40,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:40,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:41,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:42,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:43,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:44,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:45,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:45,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:46,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:28:46,501 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:28:46,502 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:28:46,504 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:28:46,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:46,971 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:28:46,989 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:28:47,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:48,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:28:48,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:28:48,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:48,991 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:28:49,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:50,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:51,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:51,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:52,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:53,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:54,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:55,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:56,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:57,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:28:57,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:58,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:28:59,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:00,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:01,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:02,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:02,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:03,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:29:03,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:29:03,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:04,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:05,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:06,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:07,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:08,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:08,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:09,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:10,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:11,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:12,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:13,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:13,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:14,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:15,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:16,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:16,974 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:29:17,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:29:17,085 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:29:17,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:29:17,087 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:29:17,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:18,015 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:29:18,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:29:18,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:29:18,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:19,016 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:29:19,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:19,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:20,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:21,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:22,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:23,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:24,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:24,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:25,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:26,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:27,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:28,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:29,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:29,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:30,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:31,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:32,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:33,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:29:33,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:29:33,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:34,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:35,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:35,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:36,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:37,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:38,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:39,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:40,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:40,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:41,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:42,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:43,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:44,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:29:44,094 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:29:44,094 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:29:44,095 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:29:44,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:45,057 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:29:45,057 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:29:45,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:46,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:46,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:46,978 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:29:47,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:48,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:29:48,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:29:48,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:49,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:50,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:51,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:51,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:52,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:53,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:54,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:55,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:56,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:57,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:29:57,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:58,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:29:59,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:00,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:01,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:02,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:03,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:30:03,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:30:03,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:03,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:04,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:05,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:06,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:07,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:08,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:08,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:09,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:10,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:11,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:12,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:13,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:14,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:30:14,200 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:30:14,200 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:30:14,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:14,201 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:30:14,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:15,082 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:30:15,082 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:30:15,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:16,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:16,982 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:30:17,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:18,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:30:18,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:30:18,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:19,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:19,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:20,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:21,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:22,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:23,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:24,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:25,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:25,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:27,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:28,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:29,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:30,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:30,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:31,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:32,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:33,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:30:33,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:30:33,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:34,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:35,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:36,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:36,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:37,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:38,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:39,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:40,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:41,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:41,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:42,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:43,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:44,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:45,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:45,752 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:30:45,752 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:30:45,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:30:45,754 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:30:46,107 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:30:46,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:46,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:46,987 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:30:47,108 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:30:47,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:48,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:30:48,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:30:48,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:49,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:50,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:51,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:52,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:52,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:53,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:54,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:55,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:56,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:57,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:58,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:30:58,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:30:59,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:00,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:01,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:02,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:03,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:31:03,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:31:03,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:03,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:04,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:05,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:06,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:07,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:08,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:09,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:09,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:10,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:11,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:12,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:13,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:14,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:14,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:15,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:16,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:16,995 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:31:17,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:17,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:31:17,412 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:31:17,412 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:31:17,414 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:31:18,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:31:18,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:31:18,143 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:31:18,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:19,144 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:31:19,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:20,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:20,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:21,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:22,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:23,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:24,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:25,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:25,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:26,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:27,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:28,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:29,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:30,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:31,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:31,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:32,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:33,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:31:33,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:31:33,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:34,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:35,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:36,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:36,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:37,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:38,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:39,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:40,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:41,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:41,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:42,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:43,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:44,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:45,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:45,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:31:45,967 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:31:45,968 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:31:45,970 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:31:46,166 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:31:46,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:46,994 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:31:46,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:47,170 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:31:47,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:48,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:31:48,032 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:31:48,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:49,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:50,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:51,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:52,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:52,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:53,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:54,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:55,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:56,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:57,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:57,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:31:58,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:31:59,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:00,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:01,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:02,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:03,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:32:03,032 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:32:03,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:03,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:04,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:05,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:06,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:07,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:08,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:08,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:09,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:10,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:11,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:12,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:13,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:13,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:14,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:15,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:16,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:17,003 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:32:17,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:18,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:32:18,043 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:32:18,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:19,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:32:19,305 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:32:19,306 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:32:19,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:19,307 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:32:19,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:20,201 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:32:20,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:21,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:32:21,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:22,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:23,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:24,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:24,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:25,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:26,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:27,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:28,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:29,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:30,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:30,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:31,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:32,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:33,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:32:33,037 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:32:33,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:34,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:35,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:35,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:36,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:37,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:38,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:39,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:40,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:40,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:41,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:42,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:43,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:44,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:45,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:46,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:46,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:47,004 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:32:47,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:48,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:32:48,044 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:32:48,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:49,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:50,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:51,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:51,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:52,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:53,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:32:53,446 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:32:53,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:32:53,448 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:32:53,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:54,231 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:32:54,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:55,232 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:32:55,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:56,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:56,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:32:57,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:58,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:32:59,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:00,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:01,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:02,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:02,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:03,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:33:03,044 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:33:03,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:04,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:05,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:06,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:07,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:07,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:08,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:09,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:10,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:11,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:12,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:13,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:13,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:14,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:15,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:16,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:17,008 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:33:17,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:18,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:33:18,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:33:18,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:19,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:19,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:20,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:21,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:22,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:23,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:24,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:24,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:25,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:33:25,395 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:33:25,396 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:33:25,398 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:33:25,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:26,261 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:33:26,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:27,262 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:33:27,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:28,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:29,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:30,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:30,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:31,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:32,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:33,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:33:33,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:33:33,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:34,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:35,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:36,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:36,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:37,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:38,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:39,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:40,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:41,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:41,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:42,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:43,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:44,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:45,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:46,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:46,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:47,012 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:33:47,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:48,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:33:48,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:33:48,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:49,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:50,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:51,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:51,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:52,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:53,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:54,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:55,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:56,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:33:56,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:57,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:58,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:33:58,150 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:33:58,150 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:33:58,152 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:33:58,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:33:58,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:33:59,298 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:33:59,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:00,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:01,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:02,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:02,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:03,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:34:03,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:34:03,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:04,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:05,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:06,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:07,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:08,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:08,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:09,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:10,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:11,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:12,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:13,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:13,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:14,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:15,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:16,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:17,016 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:34:17,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:18,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:34:18,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:34:18,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:19,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:19,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:20,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:21,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:22,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:23,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:24,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:24,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:25,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:26,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:27,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:28,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:29,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:30,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:34:30,291 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:34:30,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:34:30,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:30,293 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:34:30,334 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:34:30,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:31,335 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:34:31,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:32,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:33,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:34:33,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:34:33,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:34,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:35,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:35,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:36,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:37,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:38,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:39,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:40,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:40,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:41,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:43,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:43,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:44,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:45,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:45,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:46,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:47,020 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:34:47,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:48,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:34:48,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:34:48,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:49,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:50,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:51,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:51,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:52,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:53,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:54,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:55,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:56,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:34:56,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:57,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:58,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:34:59,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:00,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:35:00,765 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:35:00,765 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:35:00,767 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:35:01,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:01,364 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:35:01,364 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:35:02,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:02,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:03,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:35:03,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:35:03,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:04,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:05,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:06,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:07,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:07,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:08,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:09,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:10,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:11,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:12,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:13,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:13,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:14,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:15,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:16,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:17,024 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:35:17,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:18,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:35:18,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:35:18,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:18,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:19,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:20,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:21,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:22,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:23,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:23,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:24,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:25,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:26,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:27,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:28,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:28,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:29,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:30,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:31,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:32,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:33,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:35:33,049 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:35:33,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:33,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:35:33,487 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:35:33,487 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:35:33,488 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:35:33,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:34,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:34,396 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:35:35,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:35,397 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:35:36,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:37,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:38,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:39,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:39,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:40,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:41,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:42,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:43,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:44,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:45,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:45,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:46,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:47,028 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:35:47,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:48,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:35:48,050 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:35:48,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:49,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:50,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:51,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:51,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:52,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:53,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:54,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:55,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:56,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:56,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:35:57,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:58,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:35:59,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:00,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:01,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:01,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:02,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:03,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:36:03,050 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:36:03,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:04,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:36:04,051 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:36:04,051 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:36:04,053 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:36:04,425 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:36:04,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:05,426 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:36:05,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:06,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:07,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:07,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:08,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:09,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:10,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:11,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:12,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:12,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:13,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:14,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:15,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:16,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:17,032 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:36:17,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:18,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:18,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:36:18,177 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:36:18,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:19,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:20,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:21,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:22,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:23,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:23,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:24,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:25,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:26,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:27,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:28,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:28,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:29,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:30,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:31,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:32,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:33,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:36:33,085 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:36:33,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:33,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:34,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:35,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:35,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:36:35,696 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:36:35,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:36:35,698 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:36:36,457 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:36:36,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:37,459 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:36:37,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:38,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:39,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:39,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:40,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:41,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:42,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:43,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:44,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:44,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:45,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:46,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:47,037 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:36:47,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:48,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:36:48,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:36:48,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:49,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:49,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:50,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:51,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:52,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:53,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:54,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:55,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:55,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:36:56,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:57,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:58,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:36:59,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:00,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:00,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:01,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:02,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:03,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:37:03,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:37:03,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:04,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:05,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:05,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:37:05,909 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:37:05,909 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:37:05,910 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:37:05,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:06,486 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:37:06,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:07,487 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:37:07,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:08,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:09,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:10,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:11,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:11,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:12,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:13,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:14,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:15,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:16,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:17,040 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:37:17,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:17,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:18,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:37:18,089 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:37:18,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:19,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:20,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:21,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:22,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:22,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:23,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:24,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:25,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:26,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:27,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:27,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:28,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:29,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:30,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:31,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:32,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:32,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:33,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:37:33,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:37:33,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:34,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:35,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:36,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:37,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:37,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:38,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:38,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:37:38,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:37:38,693 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:37:38,694 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:37:39,519 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:37:39,519 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:37:39,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:40,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:41,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:42,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:43,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:43,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:44,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:45,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:46,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:47,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:37:47,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:48,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:37:48,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:37:48,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:48,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:49,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:50,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:51,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:52,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:53,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:54,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:54,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:55,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:56,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:57,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:58,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:37:59,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:37:59,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:00,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:01,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:02,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:03,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:38:03,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:38:03,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:04,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:05,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:05,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:06,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:07,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:08,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:09,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:10,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:10,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:38:10,181 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:38:10,182 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:38:10,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:38:10,545 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:38:10,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:11,546 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:38:11,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:12,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:13,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:14,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:15,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:15,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:16,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:17,048 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:38:17,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:18,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:38:18,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:38:18,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:19,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:20,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:21,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:21,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:22,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:23,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:24,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:25,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:26,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:26,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:27,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:28,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:29,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:30,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:31,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:32,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:32,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:33,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:38:33,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:38:33,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:34,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:35,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:36,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:37,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:37,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:38,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:39,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:39,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:38:39,823 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:38:39,823 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:38:39,824 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:38:40,575 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:38:40,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:41,576 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:38:41,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:42,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:42,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:43,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:44,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:45,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:46,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:47,052 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:38:47,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:48,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:48,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:38:48,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:38:48,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:49,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:50,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:51,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:52,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:53,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:53,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:54,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:55,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:56,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:57,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:58,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:38:58,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:38:59,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:00,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:01,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:02,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:03,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:39:03,117 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:39:03,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:03,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:04,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:05,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:06,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:07,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:08,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:08,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:39:08,753 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:39:08,753 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:39:08,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:08,754 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:39:09,604 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:39:09,605 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:39:09,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:10,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:11,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:12,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:13,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:14,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:14,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:15,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:16,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:17,056 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:39:17,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:18,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:39:18,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:39:18,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:19,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:20,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:20,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:21,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:22,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:23,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:24,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:25,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:25,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:26,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:27,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:28,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:29,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:30,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:31,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:31,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:32,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:33,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:39:33,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:39:33,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:34,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:35,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:36,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:36,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:37,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:38,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:39,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:40,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:41,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:41,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:39:41,366 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:39:41,366 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:39:41,368 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:39:41,634 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:39:41,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:42,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:43,638 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:39:43,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:44,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:45,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:46,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:47,061 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:39:47,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:47,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:48,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:39:48,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:39:48,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:49,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:50,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:51,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:52,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:52,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:53,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:54,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:55,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:56,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:57,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:58,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:39:58,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:39:59,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:00,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:01,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:02,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:03,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:40:03,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:40:03,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:03,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:04,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:05,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:06,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:07,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:08,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:09,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:09,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:10,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:11,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:12,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:40:12,388 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:40:12,388 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:40:12,389 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:40:12,671 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:40:12,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:13,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:14,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:14,673 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:40:14,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:15,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:16,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:17,065 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:40:17,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:18,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:40:18,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:40:18,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:19,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:20,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:20,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:21,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:22,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:23,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:24,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:25,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:26,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:26,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:27,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:28,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:29,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:30,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:31,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:31,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:32,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:33,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:40:33,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:40:33,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:34,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:35,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:36,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:37,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:37,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:38,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:39,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:40,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:41,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:42,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:42,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:43,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:44,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:40:44,616 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:40:44,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:40:44,620 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:40:44,702 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:40:45,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:46,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:46,705 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:40:47,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:47,067 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:40:48,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:48,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:48,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:40:48,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:40:49,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:50,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:51,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:52,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:53,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:53,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:40:54,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:55,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:56,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:57,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:58,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:59,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:40:59,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:00,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:01,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:02,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:03,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:03,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:41:03,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:41:04,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:05,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:05,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:06,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:07,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:08,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:09,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:10,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:10,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:11,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:12,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:13,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:14,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:15,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:15,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:41:15,874 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:41:15,874 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:41:15,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:15,875 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:41:16,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:16,732 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:41:17,072 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:41:17,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:18,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:41:18,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:41:18,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:18,734 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:41:19,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:20,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:21,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:21,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:22,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:23,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:24,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:25,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:26,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:27,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:27,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:28,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:29,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:30,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:31,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:32,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:33,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:41:33,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:41:33,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:33,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:34,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:35,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:36,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:37,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:38,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:39,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:39,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:40,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:41,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:42,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:43,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:44,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:45,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:45,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:46,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:47,081 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:41:47,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:48,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:41:48,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:41:48,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:49,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:41:49,053 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:41:49,053 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:41:49,054 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:41:49,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:49,761 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:41:50,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:50,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:50,762 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:41:51,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:52,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:53,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:54,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:55,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:56,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:41:56,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:57,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:58,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:41:59,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:00,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:01,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:01,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:02,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:03,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:42:03,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:42:03,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:04,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:05,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:06,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:06,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:07,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:08,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:09,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:10,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:11,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:12,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:12,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:13,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:14,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:15,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:16,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:17,089 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:42:17,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:18,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:18,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:42:18,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:42:18,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:19,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:20,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:21,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:22,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:22,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:42:22,637 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:42:22,638 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:42:22,639 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:42:22,793 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:42:23,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:23,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:24,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:24,794 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:42:25,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:26,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:27,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:28,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:29,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:29,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:30,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:31,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:32,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:33,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:42:33,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:42:33,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:34,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:35,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:35,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:36,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:37,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:38,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:39,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:40,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:40,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:41,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:42,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:43,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:44,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:45,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:46,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:46,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:47,084 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:42:47,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:48,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:42:48,114 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:42:48,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:49,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:50,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:51,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:51,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:52,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:53,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:42:53,512 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:42:53,513 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:42:53,515 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:42:53,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:42:53,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:54,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:42:54,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:55,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:56,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:57,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:42:57,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:58,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:42:59,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:00,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:01,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:02,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:03,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:43:03,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:43:03,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:03,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:04,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:05,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:06,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:07,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:08,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:09,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:09,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:10,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:11,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:12,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:13,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:14,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:15,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:15,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:16,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:17,092 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:43:17,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:18,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:43:18,116 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:43:18,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:19,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:20,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:20,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:21,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:43:21,316 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:43:21,316 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:43:21,317 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:43:21,842 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:43:21,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:22,843 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:43:22,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:23,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:24,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:25,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:26,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:26,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:27,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:28,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:29,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:30,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:31,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:32,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:32,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:33,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:43:33,114 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:43:33,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:34,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:35,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:36,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:37,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:37,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:38,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:39,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:40,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:41,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:42,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:43,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:43,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:44,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:45,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:46,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:47,093 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:43:47,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:48,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:43:48,119 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:43:48,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:48,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:49,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:50,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:43:50,700 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:43:50,700 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:43:50,701 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:43:50,866 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:43:51,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:52,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:52,868 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:43:53,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:53,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:43:54,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:55,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:56,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:57,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:58,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:59,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:43:59,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:00,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:01,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:02,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:03,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:03,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:44:03,117 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:44:04,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:04,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:05,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:06,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:07,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:08,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:09,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:10,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:10,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:11,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:12,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:13,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:14,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:15,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:15,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:16,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:17,097 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:44:17,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:18,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:18,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:44:18,127 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:44:19,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:20,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:21,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:21,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:21,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:44:21,495 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:44:21,495 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:44:21,496 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:44:21,895 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:44:22,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:22,896 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:44:23,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:24,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:25,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:26,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:26,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:27,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:28,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:29,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:30,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:31,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:31,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:32,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:33,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:33,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:44:33,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:44:34,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:35,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:36,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:37,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:37,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:38,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:39,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:40,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:41,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:42,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:42,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:43,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:44,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:45,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:46,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:47,104 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:44:47,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:48,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:48,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:44:48,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:44:48,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:49,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:50,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:51,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:52,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:52,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:44:52,532 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:44:52,532 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:44:52,533 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:44:52,929 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:44:53,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:53,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:44:54,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:54,930 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:44:55,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:56,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:57,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:58,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:59,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:44:59,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:00,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:01,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:02,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:03,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:45:03,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:45:03,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:04,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:05,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:05,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:06,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:07,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:08,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:09,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:10,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:10,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:11,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:12,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:13,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:14,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:15,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:16,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:16,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:17,103 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:45:17,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:18,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:45:18,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:45:18,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:19,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:20,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:21,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:21,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:22,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:23,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:24,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:24,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:45:24,241 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:45:24,241 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:45:24,243 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:45:24,961 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:45:25,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:26,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:26,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:26,962 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:45:27,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:28,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:29,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:30,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:31,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:32,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:32,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:33,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:45:33,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:45:33,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:34,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:35,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:36,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:37,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:37,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:38,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:39,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:40,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:41,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:42,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:42,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:43,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:44,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:45,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:46,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:47,107 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:45:47,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:48,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:48,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:45:48,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:45:48,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:49,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:50,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:51,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:52,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:53,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:53,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:54,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:55,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:56,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:57,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:58,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:45:58,066 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:45:58,067 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:45:58,068 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:45:58,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:45:58,985 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:45:59,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:45:59,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:00,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:00,987 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:46:01,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:02,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:03,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:46:03,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:46:03,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:04,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:04,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:05,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:06,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:07,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:08,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:09,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:10,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:10,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:11,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:12,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:13,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:14,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:15,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:15,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:16,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:17,111 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:46:17,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:18,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:46:18,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:46:18,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:19,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:20,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:21,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:21,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:22,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:23,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:24,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:25,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:26,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:26,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:27,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:28,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:29,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:46:29,037 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:46:29,038 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:46:29,040 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:46:29,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:30,014 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:46:30,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:31,016 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:46:31,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:31,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:32,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:33,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:46:33,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:46:33,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:34,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:35,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:36,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:37,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:37,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:38,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:39,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:40,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:41,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:42,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:43,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:43,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:44,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:45,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:46,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:47,115 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:46:47,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:48,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:46:48,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:46:48,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:49,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:49,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:50,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:51,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:52,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:53,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:54,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:54,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:55,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:56,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:57,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:58,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:46:58,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:46:58,867 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:46:58,867 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:46:58,868 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:46:59,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:46:59,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:46:59,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:00,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:01,047 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:47:01,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:02,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:03,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:47:03,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:47:03,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:04,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:05,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:05,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:06,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:07,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:08,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:09,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:10,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:11,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:11,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:12,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:13,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:14,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:15,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:16,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:17,119 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:47:17,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:17,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:18,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:47:18,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:47:18,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:19,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:20,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:21,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:22,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:22,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:23,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:24,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:25,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:26,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:27,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:27,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:28,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:29,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:47:29,439 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:47:29,440 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:47:29,442 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:47:29,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:30,072 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:47:30,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:31,073 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:47:31,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:32,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:32,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:33,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:47:33,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:47:33,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:34,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:35,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:36,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:37,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:38,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:38,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:39,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:40,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:41,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:42,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:43,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:44,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:44,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:45,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:46,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:47,123 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:47:47,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:48,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:47:48,123 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:47:48,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:49,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:50,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:50,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:51,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:52,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:53,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:54,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:55,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:55,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:47:56,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:57,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:58,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:59,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:47:59,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:47:59,641 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:47:59,642 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:47:59,643 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:48:00,100 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:48:00,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:01,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:01,101 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:48:01,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:02,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:03,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:48:03,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:48:03,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:04,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:05,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:06,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:07,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:07,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:08,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:09,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:10,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:11,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:12,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:13,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:13,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:14,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:15,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:16,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:17,127 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:48:17,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:18,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:48:18,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:48:18,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:18,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:19,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:20,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:21,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:22,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:23,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:24,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:24,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:25,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:26,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:27,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:28,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:28,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:48:28,896 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:48:28,896 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:48:28,898 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:48:29,128 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:48:29,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:29,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:30,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:31,130 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:48:31,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:32,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:33,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:48:33,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:48:33,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:34,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:35,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:35,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:36,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:37,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:38,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:39,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:40,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:40,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:41,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:42,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:43,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:44,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:45,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:45,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:46,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:47,131 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:48:47,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:48,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:48:48,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:48:48,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:49,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:50,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:51,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:51,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:52,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:53,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:54,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:55,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:56,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:48:56,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:57,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:58,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:48:59,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:48:59,125 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:48:59,126 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:48:59,128 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:48:59,159 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:48:59,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:00,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:01,161 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:49:01,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:02,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:02,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:03,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:49:03,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:49:03,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:04,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:05,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:06,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:07,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:08,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:08,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:09,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:10,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:11,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:12,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:13,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:13,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:14,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:15,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:16,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:17,135 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:49:17,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:18,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:49:18,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:49:18,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:18,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:19,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:20,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:21,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:22,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:23,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:24,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:24,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:25,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:26,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:27,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:28,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:29,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:29,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:30,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:49:30,282 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:49:30,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:49:30,284 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:49:30,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:31,201 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:49:31,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:32,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:33,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:49:33,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:49:33,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:49:33,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:34,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:34,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:35,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:36,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:37,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:38,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:39,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:40,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:40,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:41,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:42,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:43,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:44,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:45,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:45,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:46,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:47,139 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:49:47,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:48,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:49:48,124 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:49:48,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:49,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:50,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:51,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:51,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:52,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:53,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:54,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:55,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:56,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:49:56,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:57,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:58,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:49:59,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:00,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:01,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:01,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:50:01,088 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:50:01,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:50:01,090 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:50:01,235 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:50:01,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:02,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:03,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:50:03,125 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:50:03,236 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:50:03,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:04,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:05,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:06,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:06,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:07,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:08,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:09,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:10,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:11,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:12,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:12,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:13,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:14,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:15,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:16,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:17,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:17,143 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:50:17,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:18,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:50:18,125 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:50:18,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:19,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:20,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:21,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:22,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:23,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:23,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:24,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:25,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:26,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:27,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:28,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:28,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:29,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:30,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:31,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:32,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:32,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:50:32,935 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:50:32,935 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:50:32,937 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:50:33,263 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:50:33,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:50:33,345 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:50:33,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:33,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:34,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:35,265 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:50:35,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:36,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:37,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:38,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:39,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:39,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:40,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:41,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:42,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:43,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:44,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:45,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:45,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:46,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:47,146 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:50:47,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:48,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:50:48,341 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:50:48,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:49,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:50,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:50,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:51,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:52,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:53,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:54,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:55,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:56,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:50:56,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:57,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:58,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:50:59,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:00,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:01,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:01,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:02,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:03,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:51:03,342 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:51:03,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:03,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:51:03,806 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:51:03,807 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:51:03,808 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:51:04,298 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:51:04,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:05,299 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:51:05,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:06,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:07,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:07,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:08,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:09,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:10,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:11,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:12,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:13,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:13,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:14,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:15,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:16,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:17,150 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:51:17,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:18,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:18,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:51:18,537 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:51:18,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:19,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:20,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:21,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:22,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:23,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:24,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:24,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:25,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:26,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:27,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:28,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:29,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:30,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:30,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:31,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:32,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:33,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:51:33,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:51:33,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:34,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:35,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:35,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:36,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:37,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:51:37,364 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:51:37,364 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:51:37,365 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:51:37,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:38,333 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:51:38,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:39,334 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:51:39,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:40,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:40,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:41,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:42,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:43,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:44,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:45,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:46,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:46,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:47,154 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:51:47,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:48,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:51:48,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:51:48,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:49,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:50,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:51,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:51,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:52,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:53,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:54,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:55,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:56,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:57,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:51:57,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:58,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:51:59,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:00,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:01,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:02,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:03,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:03,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:52:03,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:52:03,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:04,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:05,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:06,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:07,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:08,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:08,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:09,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:10,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:10,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:52:10,835 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:52:10,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:52:10,836 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:52:11,363 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:52:11,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:12,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:13,364 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:52:13,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:13,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:14,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:15,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:16,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:17,157 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:52:17,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:18,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:52:18,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:52:18,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:19,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:19,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:20,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:21,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:22,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:23,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:24,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:25,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:25,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:26,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:27,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:28,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:29,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:30,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:31,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:31,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:32,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:33,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:52:33,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:52:33,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:34,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:35,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:36,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:36,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:37,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:38,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:39,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:40,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:41,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:41,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:52:41,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:52:41,643 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:52:41,645 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:52:42,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:42,394 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:52:43,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:43,395 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:52:44,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:45,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:46,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:47,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:47,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:47,161 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:52:48,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:48,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:52:48,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:52:49,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:50,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:51,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:52,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:53,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:53,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:54,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:55,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:56,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:57,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:58,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:52:59,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:52:59,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:00,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:01,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:02,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:03,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:03,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:53:03,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:53:04,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:04,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:05,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:06,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:07,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:08,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:09,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:10,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:10,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:11,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:12,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:13,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:13,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:53:13,707 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:53:13,708 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:53:13,709 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:53:14,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:14,439 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:53:15,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:15,440 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:53:16,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:16,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:17,166 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:53:17,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:18,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:18,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:53:18,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:53:19,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:20,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:21,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:22,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:22,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:23,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:24,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:25,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:26,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:27,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:28,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:28,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:29,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:30,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:31,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:32,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:33,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:33,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:33,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:53:33,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:53:34,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:35,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:36,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:37,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:38,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:38,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:39,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:40,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:41,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:42,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:43,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:43,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:43,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:53:43,827 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:53:43,828 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:53:43,829 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:53:44,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:44,468 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:53:45,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:45,469 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:53:46,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:47,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:53:47,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:48,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:48,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:48,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:53:48,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:53:49,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:50,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:51,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:52,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:53,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:54,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:53:54,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:55,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:56,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:57,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:58,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:53:59,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:00,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:00,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:01,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:02,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:03,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:03,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:54:03,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:54:04,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:05,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:06,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:06,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:07,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:08,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:09,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:10,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:11,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:11,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:12,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:54:12,318 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:54:12,319 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:54:12,320 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:54:12,497 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:54:12,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:13,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:13,498 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:54:14,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:15,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:16,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:17,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:17,173 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:54:17,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:18,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:54:18,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:54:18,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:19,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:20,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:21,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:22,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:23,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:23,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:24,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:25,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:26,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:27,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:28,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:29,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:29,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:30,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:31,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:32,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:33,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:54:33,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:54:33,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:34,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:34,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:35,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:36,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:37,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:38,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:39,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:40,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:40,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:41,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:42,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:43,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:44,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:54:44,537 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:54:44,537 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:54:44,538 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:54:44,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:45,527 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:54:45,528 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:54:45,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:46,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:46,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:47,177 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:54:47,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:48,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:54:48,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:54:48,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:49,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:50,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:51,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:51,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:52,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:53,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:54,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:55,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:57,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:57,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:54:58,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:54:59,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:00,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:01,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:02,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:02,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:03,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:03,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:55:03,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:55:04,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:05,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:06,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:07,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:07,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:08,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:09,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:10,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:11,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:12,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:12,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:55:12,763 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:55:12,763 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:55:12,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:12,764 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:55:13,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:13,553 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:55:13,553 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:55:14,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:15,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:16,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:17,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:17,182 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:55:18,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:18,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:18,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:55:18,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:55:19,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:20,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:21,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:22,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:23,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:24,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:24,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:25,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:26,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:27,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:28,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:29,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:29,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:30,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:31,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:32,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:33,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:33,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:55:33,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:55:34,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:34,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:35,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:36,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:37,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:38,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:39,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:40,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:40,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:41,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:42,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:42,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:55:42,856 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:55:42,856 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:55:42,857 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:55:43,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:43,585 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:55:43,585 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:55:44,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:45,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:45,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:46,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:47,187 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:55:47,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:48,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:48,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:55:48,445 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:55:49,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:50,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:51,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:51,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:52,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:53,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:54,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:55,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:56,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:55:56,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:57,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:58,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:55:59,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:00,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:01,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:02,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:02,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:03,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:03,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:56:03,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:56:04,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:05,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:06,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:07,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:07,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:08,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:09,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:10,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:11,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:11,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:56:11,894 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:56:11,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:56:11,896 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:56:12,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:12,621 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:56:12,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:13,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:13,622 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:56:14,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:15,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:16,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:17,190 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:56:17,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:18,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:18,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:18,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:56:18,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:56:19,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:20,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:21,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:22,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:23,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:24,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:24,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:25,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:26,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:27,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:28,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:29,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:29,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:30,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:31,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:32,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:33,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:33,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:56:33,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:56:34,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:34,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:35,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:36,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:37,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:38,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:39,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:40,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:40,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:41,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:41,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:56:41,341 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:56:41,341 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:56:41,343 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:56:41,657 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:56:42,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:43,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:43,659 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:56:44,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:45,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:45,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:46,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:47,194 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:56:47,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:48,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:48,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:56:48,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:56:49,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:50,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:50,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:51,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:52,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:53,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:54,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:55,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:56:55,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:56,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:57,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:58,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:56:59,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:00,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:01,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:01,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:02,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:03,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:03,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:57:03,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:57:04,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:05,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:06,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:07,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:07,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:08,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:09,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:10,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:11,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:12,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:12,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:13,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:57:13,024 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:57:13,025 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:57:13,026 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:57:13,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:13,688 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:57:13,689 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:57:14,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:15,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:16,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:17,198 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:57:17,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:17,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:18,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:18,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:57:18,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:57:19,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:20,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:21,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:22,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:23,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:23,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:24,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:25,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:26,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:27,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:28,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:29,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:29,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:30,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:31,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:32,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:33,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:33,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:57:33,446 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:57:34,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:34,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:35,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:36,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:37,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:38,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:39,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:40,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:40,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:41,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:42,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:43,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:44,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:45,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:45,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:46,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:46,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:57:46,979 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:57:46,979 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:57:46,980 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:57:47,202 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:57:47,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:47,720 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:57:47,720 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:57:48,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:48,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:57:48,447 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:57:49,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:50,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:50,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:51,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:52,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:53,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:54,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:55,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:57:55,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:56,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:57,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:58,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:57:59,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:00,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:01,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:01,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:02,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:03,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:03,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:58:03,447 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:58:04,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:05,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:06,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:06,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:07,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:08,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:09,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:10,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:11,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:12,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:12,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:13,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:14,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:15,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:16,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:17,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:17,214 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:58:17,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:18,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:58:18,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:58:18,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:19,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:58:19,238 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:58:19,238 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:58:19,240 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:58:19,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:19,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:58:19,761 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:58:20,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:21,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:22,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:22,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:23,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:24,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:25,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:26,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:27,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:28,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:28,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:29,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:30,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:31,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:32,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:33,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:33,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:58:33,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:58:33,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:34,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:35,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:36,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:37,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:38,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:38,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:39,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:40,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:41,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:42,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:43,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:44,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:44,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:45,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:46,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:47,217 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:58:47,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:48,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:58:48,455 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:58:48,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:49,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:49,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:49,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:58:49,938 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:58:49,938 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:58:49,940 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:58:50,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:50,792 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:58:51,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:51,792 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:58:52,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:53,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:54,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:55,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:58:55,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:56,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:57,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:58,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:58:59,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:00,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:01,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:01,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:02,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:03,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:59:03,459 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:59:03,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:04,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:05,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:06,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:06,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:07,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:08,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:09,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:10,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:11,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:11,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:12,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:13,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:14,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:15,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:16,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:16,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:17,214 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:59:17,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:18,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:59:18,462 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:59:18,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:19,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:20,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:21,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:21,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:21,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:59:21,812 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:59:21,812 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:59:21,818 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:59:21,823 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:59:22,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:23,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:23,825 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:59:24,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:25,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:26,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:27,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:27,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:28,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:29,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:30,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:31,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:32,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:33,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:33,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:59:33,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:59:33,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:34,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:35,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:36,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:37,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:38,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:38,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:39,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:40,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:41,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:42,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:43,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:43,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:44,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:45,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:46,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:47,219 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 15:59:47,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:48,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 15:59:48,475 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 15:59:48,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:48,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:49,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:50,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:51,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:52,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:53,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:54,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 15:59:54,296 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 15:59:54,297 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 15:59:54,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:54,303 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 15:59:54,852 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 15:59:54,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:55,853 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 15:59:55,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:56,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:57,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:58,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 15:59:59,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 15:59:59,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:00,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:01,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:02,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:03,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:00:03,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:00:03,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:04,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:04,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:05,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:06,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:07,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:08,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:09,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:10,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:10,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:11,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:12,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:13,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:14,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:15,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:16,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:16,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:17,223 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:00:17,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:18,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:00:18,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:00:18,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:19,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:20,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:21,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:21,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:22,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:23,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:24,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:25,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:26,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:00:26,426 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:00:26,427 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:00:26,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:26,432 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:00:26,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:00:27,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:27,887 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:00:28,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:29,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:30,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:31,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:32,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:32,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:33,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:33,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:00:33,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:00:34,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:35,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:36,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:37,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:38,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:38,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:39,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:40,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:41,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:42,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:43,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:43,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:44,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:45,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:46,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:47,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:00:47,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:48,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:48,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:00:48,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:00:48,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:49,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:50,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:51,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:52,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:53,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:54,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:00:54,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:55,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:55,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:00:55,884 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:00:55,884 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:00:55,886 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:00:55,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:00:56,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:57,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:57,915 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:00:58,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:59,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:00:59,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:00,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:01,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:02,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:03,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:03,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:01:03,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:01:04,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:05,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:05,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:06,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:07,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:08,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:09,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:10,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:10,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:11,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:12,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:13,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:14,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:15,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:16,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:16,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:17,232 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:01:17,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:18,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:18,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:01:18,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:01:19,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:20,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:21,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:21,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:22,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:23,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:24,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:25,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:26,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:27,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:01:27,321 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:01:27,322 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:01:27,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:27,324 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:01:27,947 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:01:27,947 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:01:28,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:28,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:29,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:30,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:31,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:32,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:32,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:33,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:01:33,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:01:33,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:34,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:35,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:36,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:37,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:38,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:38,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:39,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:40,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:41,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:42,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:43,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:43,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:44,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:45,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:46,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:47,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:01:47,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:48,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:01:48,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:01:48,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:48,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:49,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:50,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:51,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:52,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:53,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:54,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:01:54,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:55,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:56,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:57,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:01:57,133 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:01:57,134 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:01:57,135 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:01:57,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:01:57,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:01:58,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:01:59,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:00,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:00,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:01,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:02,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:03,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:03,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:02:03,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:02:04,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:05,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:05,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:06,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:07,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:08,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:09,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:10,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:10,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:11,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:12,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:13,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:14,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:15,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:16,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:16,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:17,239 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:02:17,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:18,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:18,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:02:18,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:02:19,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:20,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:21,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:21,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:22,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:23,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:24,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:25,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:26,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:26,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:27,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:02:27,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:02:27,302 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:02:27,303 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:02:28,010 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:02:28,011 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:02:28,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:29,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:30,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:31,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:32,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:32,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:33,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:02:33,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:02:33,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:34,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:35,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:36,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:37,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:38,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:38,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:39,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:40,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:41,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:42,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:43,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:43,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:44,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:45,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:46,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:47,243 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:02:47,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:48,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:02:48,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:02:48,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:48,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:49,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:50,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:51,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:52,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:53,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:54,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:54,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:55,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:56,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:57,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:58,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:02:59,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:02:59,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:00,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:03:00,637 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:03:00,638 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:03:00,643 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:03:01,037 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:03:01,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:01,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:02,038 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:03:02,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:03,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:03:03,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:03:03,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:04,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:04,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:05,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:06,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:07,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:08,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:09,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:10,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:10,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:11,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:12,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:13,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:14,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:15,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:15,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:16,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:17,246 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:03:17,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:18,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:03:18,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:03:18,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:19,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:20,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:21,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:21,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:22,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:23,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:24,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:25,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:26,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:26,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:27,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:28,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:29,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:30,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:31,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:03:31,249 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:03:31,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:03:31,254 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:03:31,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:32,069 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:03:32,069 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:03:32,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:32,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:33,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:03:33,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:03:33,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:34,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:35,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:36,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:37,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:37,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:38,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:39,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:40,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:41,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:42,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:43,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:43,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:44,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:45,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:46,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:47,250 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:03:47,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:48,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:03:48,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:03:48,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:48,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:49,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:50,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:51,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:52,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:53,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:54,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:54,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:55,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:56,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:57,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:58,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:03:59,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:03:59,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:00,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:01,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:04:01,433 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:04:01,434 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:04:01,439 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:04:01,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:02,097 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:04:02,097 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:04:02,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:03,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:04:03,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:04:03,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:04,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:04,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:05,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:06,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:07,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:08,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:09,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:10,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:10,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:11,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:12,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:13,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:14,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:15,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:15,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:16,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:17,253 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:04:17,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:18,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:04:18,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:04:18,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:19,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:20,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:21,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:21,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:22,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:23,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:24,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:25,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:26,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:27,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:27,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:28,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:29,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:30,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:31,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:04:31,488 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:04:31,488 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:04:31,493 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:04:31,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:32,123 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:04:32,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:32,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:33,124 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:04:33,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:04:33,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:04:33,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:34,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:35,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:36,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:37,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:38,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:38,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:39,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:40,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:41,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:42,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:43,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:44,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:44,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:45,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:46,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:47,257 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:04:47,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:48,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:04:48,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:04:48,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:49,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:49,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:50,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:51,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:52,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:53,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:54,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:55,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:04:55,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:56,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:57,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:58,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:04:59,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:00,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:00,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:01,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:02,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:03,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:05:03,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:05:03,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:04,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:05:04,255 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:05:04,255 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:05:04,257 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:05:05,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:05,154 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:05:05,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:06,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:07,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:07,156 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:05:08,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:09,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:10,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:10,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:11,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:12,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:13,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:14,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:15,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:16,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:16,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:17,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:17,261 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:05:18,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:18,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:05:18,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:05:19,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:20,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:21,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:21,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:22,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:23,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:24,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:25,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:26,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:26,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:27,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:28,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:29,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:30,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:31,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:31,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:05:31,472 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:05:31,473 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:05:31,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:31,474 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:05:32,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:32,184 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:05:33,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:33,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:05:33,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:05:33,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:05:34,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:35,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:36,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:37,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:37,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:38,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:39,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:40,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:41,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:42,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:43,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:43,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:44,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:45,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:46,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:47,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:47,265 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:05:48,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:48,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:05:48,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:05:48,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:49,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:50,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:51,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:52,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:53,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:54,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:54,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:05:55,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:56,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:57,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:58,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:59,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:05:59,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:00,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:01,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:01,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:06:01,588 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:06:01,589 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:06:01,590 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:06:02,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:02,214 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:06:03,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:03,215 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:06:03,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:06:03,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:06:04,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:04,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:05,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:06,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:07,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:08,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:09,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:10,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:10,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:11,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:12,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:13,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:14,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:15,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:15,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:16,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:17,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:17,270 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:06:18,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:18,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:06:18,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:06:19,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:20,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:20,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:21,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:22,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:23,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:24,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:25,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:26,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:26,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:27,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:28,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:29,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:30,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:31,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:31,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:32,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:33,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:33,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:06:33,469 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:06:34,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:34,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:06:34,231 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:06:34,231 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:06:34,233 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:06:34,251 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:06:35,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:36,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:37,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:37,253 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:06:37,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:38,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:39,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:40,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:41,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:42,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:43,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:43,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:44,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:45,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:46,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:47,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:47,274 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:06:48,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:48,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:06:48,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:06:48,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:49,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:50,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:51,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:52,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:53,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:54,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:54,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:06:55,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:56,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:57,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:58,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:59,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:06:59,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:00,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:01,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:02,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:03,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:03,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:07:03,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:07:04,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:04,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:05,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:06,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:07,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:07,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:07:07,643 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:07:07,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:07:07,645 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:07:08,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:08,281 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:07:09,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:09,282 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:07:09,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:10,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:11,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:12,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:13,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:14,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:15,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:15,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:16,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:17,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:17,276 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:07:18,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:18,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:07:18,470 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:07:19,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:20,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:20,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:21,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:22,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:23,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:24,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:25,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:26,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:26,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:27,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:28,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:29,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:30,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:31,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:32,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:32,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:33,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:33,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:07:33,478 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:07:34,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:35,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:36,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:37,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:37,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:37,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:07:37,965 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:07:37,966 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:07:37,967 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:07:38,311 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:07:38,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:39,312 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:07:39,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:40,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:41,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:42,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:43,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:43,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:44,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:45,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:46,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:47,281 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:07:47,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:48,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:48,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:07:48,473 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:07:48,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:49,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:50,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:51,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:52,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:53,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:54,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:07:54,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:55,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:56,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:57,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:58,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:07:59,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:00,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:00,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:01,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:02,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:03,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:03,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:08:03,479 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:08:04,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:05,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:05,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:06,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:07,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:08,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:08:08,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:08:08,236 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:08:08,237 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:08:08,337 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:08:08,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:09,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:10,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:11,339 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:08:11,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:11,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:12,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:13,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:14,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:15,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:16,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:17,285 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:08:17,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:17,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:18,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:08:18,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:08:18,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:19,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:20,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:21,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:22,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:22,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:23,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:24,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:25,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:26,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:27,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:27,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:28,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:29,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:30,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:31,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:32,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:32,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:33,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:08:33,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:08:33,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:34,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:35,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:36,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:37,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:37,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:38,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:39,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:40,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:41,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:42,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:08:42,373 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:08:42,373 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:08:42,375 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:08:42,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:43,371 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:08:43,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:43,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:44,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:45,373 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:08:45,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:46,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:47,289 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:08:47,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:48,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:08:48,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:08:48,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:48,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:49,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:50,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:51,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:52,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:53,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:54,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:08:54,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:55,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:56,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:57,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:58,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:08:59,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:00,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:00,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:01,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:02,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:03,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:09:03,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:09:03,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:04,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:05,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:05,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:06,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:07,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:08,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:09,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:10,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:09:10,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:09:10,044 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:09:10,045 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:09:10,405 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:09:10,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:10,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:11,406 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:09:11,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:12,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:13,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:14,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:15,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:16,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:16,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:17,293 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:09:17,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:18,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:09:18,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:09:18,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:19,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:20,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:21,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:22,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:22,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:23,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:24,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:25,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:26,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:27,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:27,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:28,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:29,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:30,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:31,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:32,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:33,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:33,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:09:33,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:09:33,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:34,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:35,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:36,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:37,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:38,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:38,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:39,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:40,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:41,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:09:41,510 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:09:41,511 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:09:41,512 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:09:41,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:42,435 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:09:42,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:43,436 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:09:43,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:43,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:44,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:45,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:46,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:47,297 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:09:47,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:48,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:09:48,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:09:48,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:48,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:49,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:50,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:51,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:52,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:53,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:54,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:09:54,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:55,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:56,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:57,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:58,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:09:59,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:00,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:00,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:01,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:02,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:03,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:10:03,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:10:03,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:04,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:05,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:05,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:06,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:07,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:08,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:09,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:10:09,889 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:10:09,889 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:10:09,890 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:10:10,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:10,464 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:10:11,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:11,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:11,464 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:10:12,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:13,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:14,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:15,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:16,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:17,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:17,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:10:17,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:18,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:18,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:10:18,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:10:19,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:20,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:21,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:22,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:22,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:23,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:24,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:25,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:26,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:27,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:27,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:28,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:29,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:30,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:31,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:32,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:33,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:33,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:33,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:10:33,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:10:34,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:35,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:36,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:37,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:38,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:38,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:39,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:40,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:41,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:41,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:10:41,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:10:41,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:10:41,046 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:10:41,495 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:10:42,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:43,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:43,497 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:10:44,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:44,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:45,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:46,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:47,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:47,305 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:10:48,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:48,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:10:48,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:10:49,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:49,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:50,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:51,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:52,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:53,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:54,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:55,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:55,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:10:56,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:57,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:58,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:10:59,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:00,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:01,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:01,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:02,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:03,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:03,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:11:03,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:11:04,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:05,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:06,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:06,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:07,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:08,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:09,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:10,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:11,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:11,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:11,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:11:11,809 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:11:11,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:11:11,811 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:11:12,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:12,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:11:13,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:13,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:11:14,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:15,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:16,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:17,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:17,309 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:11:17,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:18,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:18,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:11:18,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:11:19,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:20,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:21,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:22,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:22,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:23,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:24,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:25,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:26,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:27,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:27,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:28,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:29,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:30,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:31,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:32,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:33,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:33,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:33,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:11:33,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:11:34,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:35,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:36,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:37,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:38,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:38,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:39,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:40,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:41,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:41,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:11:41,658 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:11:41,658 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:11:41,659 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:11:42,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:42,557 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:11:43,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:43,558 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:11:43,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:44,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:45,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:46,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:47,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:47,314 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:11:48,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:48,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:11:48,481 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:11:49,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:49,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:50,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:51,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:52,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:53,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:54,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:55,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:55,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:11:56,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:57,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:58,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:11:59,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:00,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:00,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:01,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:02,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:03,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:03,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:12:03,487 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:12:04,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:05,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:05,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:06,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:07,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:08,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:09,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:12:09,204 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:12:09,205 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:12:09,206 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:12:09,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:12:09,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:10,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:11,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:11,586 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:12:11,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:12,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:13,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:14,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:15,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:16,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:17,317 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:12:17,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:17,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:18,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:12:18,485 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:12:18,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:19,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:20,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:21,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:22,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:22,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:23,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:24,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:25,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:26,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:27,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:28,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:28,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:29,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:30,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:31,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:32,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:33,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:33,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:12:33,498 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:12:33,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:34,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:35,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:36,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:37,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:37,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:12:37,977 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:12:37,977 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:12:37,979 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:12:38,617 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:12:38,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:39,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:39,618 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:12:39,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:40,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:41,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:42,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:43,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:44,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:44,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:45,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:46,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:47,321 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:12:47,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:48,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:12:48,489 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:12:48,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:49,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:49,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:50,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:51,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:52,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:53,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:54,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:12:54,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:55,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:56,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:57,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:58,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:12:59,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:00,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:00,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:01,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:02,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:03,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:13:03,495 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:13:03,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:04,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:05,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:05,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:06,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:07,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:13:07,137 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:13:07,137 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:13:07,138 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:13:07,649 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:13:07,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:08,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:09,650 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:13:09,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:10,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:10,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:11,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:12,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:13,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:14,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:15,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:15,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:16,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:17,325 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:13:17,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:18,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:13:18,489 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:13:18,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:19,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:20,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:21,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:21,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:22,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:23,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:24,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:25,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:26,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:26,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:27,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:28,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:29,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:30,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:31,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:32,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:32,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:33,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:13:33,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:13:33,750 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:34,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:35,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:36,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:37,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:37,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:38,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:39,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:40,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:13:40,356 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:13:40,357 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:13:40,358 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:13:40,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:13:40,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:41,681 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:13:41,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:42,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:42,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:43,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:44,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:45,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:46,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:47,329 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:13:47,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:48,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:48,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:13:48,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:13:48,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:49,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:50,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:51,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:52,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:53,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:53,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:54,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:55,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:56,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:57,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:58,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:13:59,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:13:59,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:00,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:01,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:02,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:03,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:14:03,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:14:03,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:04,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:05,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:05,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:06,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:07,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:08,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:09,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:10,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:10,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:11,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:12,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:13,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:14,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:15,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:15,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:16,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:14:16,600 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:14:16,600 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:14:16,602 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:14:16,718 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:14:16,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:17,333 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:14:17,719 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:14:17,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:18,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:14:18,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:14:18,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:19,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:20,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:21,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:21,995 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:22,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:23,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:24,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:25,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:26,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:27,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:27,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:28,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:29,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:30,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:31,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:32,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:32,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:33,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:14:33,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:14:33,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:34,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:35,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:36,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:37,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:38,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:38,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:40,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:41,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:42,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:43,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:43,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:44,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:44,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:14:44,971 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:14:44,972 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:14:44,973 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:14:45,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:45,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:14:46,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:47,337 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:14:47,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:47,750 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:14:48,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:48,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:14:48,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:14:49,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:49,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:50,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:51,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:52,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:53,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:54,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:54,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:14:55,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:56,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:57,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:58,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:14:59,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:00,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:00,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:01,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:02,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:03,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:03,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:15:03,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:15:04,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:05,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:06,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:06,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:07,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:08,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:09,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:10,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:11,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:11,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:12,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:13,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:14,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:14,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:15:14,989 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:15:14,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:15:14,990 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:15:15,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:15,776 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:15:16,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:17,340 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:15:17,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:17,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:17,778 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:15:18,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:18,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:15:18,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:15:19,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:20,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:21,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:22,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:22,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:23,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:24,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:25,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:26,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:27,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:28,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:28,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:29,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:30,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:31,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:32,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:33,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:33,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:15:33,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:15:34,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:34,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:35,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:36,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:37,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:38,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:39,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:39,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:40,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:41,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:42,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:43,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:44,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:44,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:45,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:46,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:47,345 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:15:47,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:48,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:15:48,421 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:15:48,422 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:15:48,423 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:15:48,806 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:15:48,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:15:48,841 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:15:48,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:49,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:49,807 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:15:50,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:50,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:51,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:52,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:53,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:54,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:55,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:55,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:15:56,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:57,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:58,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:15:59,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:00,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:00,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:01,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:02,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:03,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:03,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:16:03,840 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:16:04,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:05,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:05,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:06,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:07,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:08,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:09,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:10,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:11,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:11,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:12,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:13,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:14,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:15,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:16,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:16,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:17,349 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:16:17,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:18,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:18,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:16:18,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:16:19,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:20,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:21,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:16:21,403 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:16:21,403 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:16:21,405 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:16:21,834 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:16:21,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:22,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:22,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:23,836 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:16:23,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:24,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:25,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:26,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:27,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:27,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:28,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:29,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:30,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:31,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:32,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:33,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:33,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:16:33,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:16:33,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:34,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:35,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:36,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:37,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:38,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:39,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:39,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:40,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:41,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:42,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:43,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:44,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:44,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:45,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:46,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:47,356 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:16:47,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:48,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:16:48,850 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:16:48,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:49,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:49,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:50,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:16:50,819 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:16:50,820 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:16:50,822 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:16:50,860 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:16:51,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:51,861 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:16:51,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:52,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:53,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:54,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:55,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:16:55,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:56,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:57,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:58,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:16:59,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:00,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:00,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:01,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:02,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:03,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:17:03,846 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:17:03,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:04,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:05,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:05,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:06,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:07,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:08,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:09,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:10,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:11,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:11,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:12,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:13,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:14,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:15,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:16,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:16,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:17,367 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:17:17,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:18,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:17:18,858 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:17:18,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:19,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:20,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:21,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:21,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:22,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:17:22,048 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:17:22,048 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:17:22,050 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:17:22,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:17:22,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:23,887 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:17:23,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:24,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:25,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:26,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:27,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:27,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:28,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:29,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:30,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:31,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:32,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:33,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:33,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:17:33,850 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:17:33,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:34,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:35,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:36,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:37,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:38,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:38,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:39,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:40,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:41,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:42,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:43,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:44,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:44,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:45,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:46,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:47,364 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:17:47,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:48,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:17:48,858 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:17:48,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:49,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:49,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:50,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:51,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:52,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:17:52,835 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:17:52,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:17:52,840 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:17:52,915 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:17:53,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:53,916 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:17:54,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:55,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:55,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:17:56,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:57,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:58,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:17:59,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:00,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:01,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:01,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:02,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:03,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:03,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:18:03,855 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:18:04,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:05,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:06,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:06,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:07,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:08,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:09,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:10,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:11,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:12,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:12,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:13,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:14,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:15,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:16,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:17,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:17,368 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:18:18,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:18,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:18,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:18:18,862 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:18:19,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:20,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:21,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:22,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:23,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:23,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:24,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:18:24,206 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:18:24,206 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:18:24,212 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:18:24,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:24,943 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:18:25,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:25,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:18:26,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:27,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:28,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:29,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:29,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:30,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:31,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:32,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:33,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:33,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:18:33,859 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:18:34,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:34,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:35,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:36,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:37,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:38,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:39,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:40,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:40,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:41,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:42,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:43,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:44,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:45,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:46,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:46,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:47,378 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:18:47,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:48,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:48,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:18:48,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:18:49,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:50,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:51,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:51,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:52,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:53,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:18:53,181 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:18:53,181 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:18:53,182 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:18:53,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:53,976 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:18:54,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:55,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:55,978 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:18:56,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:57,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:57,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:18:58,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:18:59,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:00,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:01,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:02,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:02,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:03,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:03,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:19:03,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:19:04,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:05,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:06,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:07,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:08,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:08,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:09,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:10,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:11,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:12,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:13,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:13,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:14,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:15,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:16,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:17,374 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:19:17,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:18,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:18,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:19:18,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:19:18,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:19,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:20,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:21,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:22,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:23,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:24,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:24,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:19:24,800 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:19:24,800 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:19:24,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:24,802 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:19:25,009 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:19:25,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:26,010 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:19:26,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:27,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:28,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:29,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:29,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:30,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:31,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:32,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:33,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:33,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:19:33,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:19:34,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:35,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:35,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:36,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:37,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:38,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:39,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:40,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:40,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:41,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:42,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:43,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:44,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:45,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:46,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:46,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:47,378 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:19:47,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:48,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:48,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:19:48,868 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:19:49,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:50,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:51,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:51,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:52,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:53,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:53,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:19:53,943 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:19:53,943 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:19:53,945 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:19:54,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:19:54,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:55,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:56,047 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:19:56,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:57,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:57,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:19:58,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:19:59,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:00,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:01,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:02,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:02,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:03,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:03,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:20:03,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:20:04,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:05,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:06,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:07,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:08,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:08,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:09,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:10,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:11,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:12,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:13,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:13,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:14,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:15,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:16,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:17,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:20:17,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:18,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:18,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:20:18,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:20:18,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:19,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:20,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:21,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:22,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:23,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:24,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:24,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:25,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:20:25,035 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:20:25,036 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:20:25,037 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:20:25,072 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:20:25,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:26,073 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:20:26,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:27,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:28,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:29,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:30,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:30,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:31,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:32,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:33,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:33,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:20:33,866 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:20:34,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:35,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:36,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:36,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:37,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:38,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:39,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:40,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:41,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:41,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:42,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:43,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:44,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:45,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:46,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:46,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:47,387 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:20:47,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:48,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:48,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:20:48,871 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:20:49,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:50,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:51,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:51,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:52,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:53,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:53,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:20:53,927 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:20:53,928 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:20:53,929 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:20:54,107 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:20:54,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:55,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:56,108 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:20:56,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:57,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:57,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:20:58,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:20:59,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:00,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:01,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:02,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:03,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:03,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:03,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:21:03,869 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:21:04,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:05,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:06,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:07,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:08,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:08,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:09,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:10,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:11,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:12,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:13,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:13,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:14,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:15,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:16,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:17,391 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:21:17,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:18,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:18,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:21:18,875 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:21:18,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:19,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:20,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:21,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:22,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:23,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:24,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:21:24,473 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:21:24,473 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:21:24,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:24,475 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:21:24,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:25,137 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:21:25,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:26,138 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:21:26,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:27,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:28,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:29,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:29,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:30,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:31,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:32,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:33,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:21:33,874 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:21:33,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:34,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:34,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:35,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:36,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:37,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:38,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:39,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:40,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:40,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:41,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:42,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:43,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:44,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:45,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:46,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:46,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:47,395 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:21:47,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:48,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:21:48,877 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:21:48,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:49,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:50,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:51,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:51,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:52,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:53,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:21:53,660 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:21:53,661 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:21:53,662 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:21:54,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:54,174 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:21:54,174 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:21:55,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:56,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:57,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:57,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:21:58,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:21:59,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:00,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:01,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:02,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:03,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:03,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:03,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:22:03,877 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:22:04,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:05,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:06,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:07,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:08,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:09,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:09,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:10,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:11,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:12,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:13,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:14,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:14,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:15,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:16,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:17,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:17,399 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:22:18,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:18,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:22:18,887 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:22:19,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:20,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:20,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:21,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:22,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:23,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:23,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:22:23,824 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:22:23,825 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:22:23,826 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:22:24,205 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:22:24,206 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:22:24,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:25,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:25,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:26,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:27,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:28,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:29,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:30,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:31,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:31,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:32,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:33,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:33,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:22:33,881 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:22:34,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:35,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:36,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:37,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:37,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:38,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:39,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:40,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:41,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:42,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:42,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:43,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:44,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:45,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:46,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:47,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:47,404 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:22:48,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:48,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:48,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:22:48,882 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:22:49,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:50,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:51,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:52,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:53,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:53,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:22:53,703 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:22:53,703 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:22:53,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:53,705 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:22:54,233 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:22:54,234 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:22:54,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:55,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:56,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:57,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:58,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:22:58,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:22:59,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:00,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:01,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:02,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:03,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:03,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:23:03,882 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:23:03,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:04,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:05,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:06,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:07,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:08,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:09,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:09,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:10,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:11,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:12,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:13,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:14,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:15,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:15,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:16,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:17,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:17,406 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:23:18,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:18,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:23:18,884 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:23:19,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:20,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:20,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:21,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:22,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:23,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:23,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:23:23,448 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:23:23,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:23:23,450 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:23:24,264 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:23:24,264 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:23:24,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:25,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:26,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:26,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:27,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:28,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:29,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:30,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:31,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:32,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:32,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:33,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:33,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:23:33,882 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:23:34,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:35,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:36,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:37,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:37,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:38,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:39,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:40,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:41,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:42,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:43,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:43,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:44,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:45,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:46,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:47,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:47,408 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:23:48,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:48,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:23:48,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:23:48,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:49,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:50,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:51,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:52,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:53,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:53,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:23:53,425 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:23:53,425 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:23:53,430 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:23:54,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:54,296 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:23:54,296 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:23:54,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:23:55,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:56,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:57,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:58,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:59,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:23:59,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:00,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:01,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:02,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:03,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:03,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:24:03,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:24:04,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:04,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:05,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:06,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:07,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:08,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:09,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:10,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:10,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:11,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:12,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:13,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:14,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:15,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:16,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:16,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:17,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:17,412 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:24:18,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:18,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:24:18,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:24:19,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:20,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:21,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:21,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:22,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:23,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:23,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:24:23,562 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:24:23,563 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:24:23,565 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:24:24,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:24,326 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:24:24,327 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:24:25,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:26,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:27,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:27,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:28,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:29,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:30,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:31,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:32,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:33,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:33,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:33,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:24:33,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:24:34,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:35,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:36,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:37,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:38,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:39,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:39,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:40,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:41,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:42,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:43,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:44,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:44,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:45,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:46,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:47,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:47,416 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:24:48,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:48,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:24:48,887 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:24:49,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:50,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:50,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:51,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:52,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:53,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:54,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:24:54,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:24:54,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:24:54,091 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:24:54,357 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:24:54,357 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:24:54,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:55,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:24:55,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:56,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:57,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:58,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:24:59,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:00,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:01,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:01,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:02,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:03,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:03,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:25:03,887 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:25:04,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:05,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:06,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:07,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:07,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:08,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:09,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:10,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:11,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:12,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:12,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:13,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:14,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:15,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:16,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:17,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:17,421 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:25:17,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:18,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:18,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:25:18,889 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:25:19,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:20,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:21,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:22,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:22,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:23,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:24,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:25:24,156 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:25:24,156 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:25:24,161 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:25:24,389 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:25:24,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:25,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:26,391 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:25:26,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:27,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:28,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:28,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:29,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:30,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:31,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:32,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:33,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:33,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:25:33,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:25:34,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:34,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:35,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:36,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:37,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:38,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:39,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:40,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:40,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:41,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:42,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:43,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:44,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:45,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:45,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:46,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:47,424 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:25:47,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:48,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:48,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:25:48,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:25:49,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:50,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:50,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:51,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:52,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:53,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:54,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:55,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:56,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:25:56,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:25:56,031 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:25:56,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:25:56,036 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:25:56,426 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:25:56,427 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:25:57,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:58,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:25:59,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:00,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:01,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:01,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:02,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:03,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:03,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:26:03,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:26:04,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:05,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:06,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:07,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:07,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:08,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:09,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:10,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:11,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:12,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:12,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:13,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:14,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:15,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:16,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:17,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:17,427 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:26:17,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:18,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:18,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:26:18,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:26:19,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:20,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:21,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:22,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:23,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:23,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:24,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:25,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:26,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:26:26,102 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:26:26,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:26:26,107 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:26:26,456 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:26:26,457 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:26:27,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:27,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:28,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:28,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:29,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:30,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:31,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:32,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:33,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:33,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:26:33,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:26:34,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:34,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:35,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:36,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:37,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:38,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:39,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:40,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:40,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:41,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:42,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:43,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:44,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:45,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:45,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:46,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:47,431 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:26:47,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:48,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:48,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:26:48,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:26:49,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:50,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:50,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:51,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:52,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:53,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:54,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:55,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:56,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:26:56,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:57,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:58,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:59,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:26:59,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:26:59,965 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:26:59,966 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:26:59,971 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:27:00,491 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:27:00,491 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:27:00,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:01,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:01,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:02,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:03,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:03,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:27:03,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:27:04,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:05,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:06,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:07,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:07,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:08,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:09,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:10,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:11,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:12,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:12,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:13,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:14,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:15,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:16,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:17,435 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:27:17,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:17,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:18,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:18,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:27:18,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:27:19,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:20,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:21,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:22,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:23,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:23,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:24,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:25,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:26,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:27,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:28,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:28,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:28,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:27:28,593 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:27:28,593 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:27:28,595 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:27:29,529 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:27:29,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:30,530 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:27:30,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:31,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:32,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:33,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:33,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:27:33,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:27:34,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:34,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:35,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:36,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:37,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:38,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:39,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:39,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:40,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:41,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:42,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:43,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:44,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:44,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:45,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:46,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:47,439 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:27:47,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:48,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:48,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:27:48,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:27:49,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:49,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:50,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:51,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:52,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:53,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:54,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:55,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:27:55,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:56,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:57,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:58,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:27:58,264 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:27:58,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:27:58,266 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:27:58,562 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:27:58,562 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:27:58,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:27:59,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:00,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:00,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:01,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:02,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:03,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:03,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:28:03,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:28:04,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:05,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:06,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:06,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:07,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:08,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:09,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:10,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:11,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:11,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:12,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:13,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:14,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:15,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:16,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:16,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:17,444 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:28:17,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:18,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:18,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:28:18,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:28:19,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:20,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:21,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:21,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:22,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:23,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:24,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:25,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:26,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:26,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:27,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:28,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:29,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:28:29,569 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:28:29,569 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:28:29,571 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:28:29,590 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:28:29,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:30,591 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:28:30,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:31,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:31,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:32,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:33,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:28:33,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:28:33,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:34,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:35,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:36,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:37,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:37,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:38,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:39,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:40,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:41,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:42,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:42,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:43,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:44,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:45,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:46,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:47,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:47,448 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:28:47,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:48,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:28:48,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:28:48,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:49,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:50,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:51,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:52,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:53,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:53,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:54,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:55,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:56,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:57,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:58,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:28:58,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:28:59,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:00,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:01,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:02,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:29:02,075 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:29:02,075 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:29:02,076 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:29:02,622 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:29:02,622 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:29:02,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:03,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:03,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:29:03,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:29:03,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:04,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:05,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:06,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:07,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:08,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:09,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:09,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:10,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:11,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:12,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:13,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:14,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:15,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:15,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:16,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:17,452 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:29:17,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:18,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:29:18,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:29:18,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:19,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:20,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:20,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:21,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:22,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:23,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:24,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:25,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:26,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:26,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:27,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:28,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:29,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:30,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:31,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:31,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:29:31,397 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:29:31,397 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:29:31,398 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:29:31,651 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:29:31,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:32,652 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:29:32,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:33,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:29:33,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:29:33,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:34,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:35,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:36,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:37,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:37,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:38,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:39,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:40,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:41,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:42,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:43,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:43,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:45,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:46,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:47,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:47,455 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:29:48,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:48,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:48,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:29:48,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:29:49,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:50,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:51,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:52,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:53,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:54,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:54,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:55,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:56,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:57,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:58,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:29:59,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:29:59,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:00,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:01,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:02,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:02,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:30:02,886 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:30:02,887 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:30:02,888 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:30:03,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:03,684 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:30:03,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:30:03,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:30:04,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:04,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:05,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:05,685 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:30:06,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:07,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:08,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:09,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:10,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:10,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:11,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:12,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:13,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:14,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:15,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:16,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:16,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:17,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:17,460 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:30:18,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:18,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:30:18,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:30:19,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:20,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:21,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:22,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:22,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:23,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:24,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:25,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:26,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:27,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:27,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:28,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:29,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:30,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:31,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:32,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:33,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:33,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:30:33,220 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:30:33,221 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:30:33,222 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:30:33,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:33,716 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:30:33,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:30:33,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:30:34,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:35,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:35,717 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:30:36,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:37,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:38,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:39,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:39,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:40,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:41,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:42,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:43,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:44,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:44,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:45,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:46,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:47,465 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:30:47,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:48,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:48,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:30:48,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:30:49,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:50,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:50,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:51,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:52,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:53,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:54,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:55,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:30:55,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:56,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:57,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:58,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:30:59,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:00,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:00,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:01,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:02,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:31:02,480 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:31:02,480 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:31:02,481 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:31:02,747 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:31:02,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:03,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:03,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:31:03,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:31:03,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:31:04,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:05,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:06,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:06,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:07,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:08,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:09,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:10,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:11,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:11,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:12,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:13,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:14,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:15,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:16,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:16,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:17,468 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:31:17,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:18,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:18,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:31:18,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:31:19,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:20,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:21,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:22,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:22,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:23,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:24,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:25,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:26,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:27,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:27,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:28,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:29,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:30,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:31,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:31:31,565 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:31:31,565 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:31:31,567 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:31:31,785 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:31:31,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:32,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:32,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:33,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:33,786 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:31:33,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:31:33,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:31:34,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:35,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:36,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:37,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:38,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:38,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:39,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:40,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:41,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:42,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:43,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:44,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:44,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:45,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:46,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:47,472 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:31:47,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:48,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:48,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:31:48,892 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:31:49,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:49,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:50,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:51,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:52,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:53,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:54,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:55,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:31:55,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:56,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:57,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:58,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:31:59,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:00,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:00,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:32:00,654 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:32:00,655 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:32:00,656 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:32:00,822 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:32:01,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:01,823 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:32:02,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:03,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:03,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:32:03,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:32:04,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:05,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:05,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:06,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:07,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:08,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:09,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:10,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:11,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:11,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:12,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:13,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:14,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:15,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:16,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:16,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:17,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:17,476 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:32:18,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:18,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:32:18,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:32:19,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:20,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:21,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:22,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:22,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:23,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:24,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:25,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:26,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:27,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:27,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:28,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:29,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:30,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:31,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:32,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:32,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:32:32,712 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:32:32,712 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:32:32,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:32,713 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:32:32,854 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:32:33,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:33,855 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:32:33,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:32:33,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:32:34,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:35,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:36,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:37,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:38,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:38,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:39,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:40,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:41,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:42,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:43,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:44,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:44,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:45,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:46,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:47,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:47,480 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:32:48,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:48,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:32:48,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:32:49,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:50,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:50,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:51,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:52,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:53,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:54,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:55,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:32:55,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:56,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:57,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:58,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:32:59,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:00,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:00,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:01,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:01,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:33:01,444 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:33:01,444 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:33:01,445 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:33:01,889 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:33:02,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:03,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:03,891 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:33:03,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:33:03,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:33:04,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:05,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:06,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:06,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:07,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:08,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:09,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:10,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:11,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:11,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:12,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:13,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:14,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:15,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:16,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:16,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:17,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:17,484 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:33:18,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:18,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:33:18,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:33:19,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:20,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:21,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:21,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:22,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:23,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:24,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:25,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:26,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:27,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:27,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:28,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:29,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:30,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:31,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:32,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:32,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:33,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:33,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:33:33,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:33:33,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:33:34,053 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:33:34,054 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:33:34,055 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:33:34,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:34,923 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:33:35,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:35,924 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:33:36,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:37,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:38,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:38,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:39,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:40,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:41,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:42,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:43,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:44,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:44,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:45,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:46,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:47,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:47,489 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:33:48,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:48,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:33:48,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:33:49,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:50,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:50,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:51,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:52,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:53,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:54,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:55,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:33:55,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:56,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:57,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:58,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:33:59,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:00,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:00,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:01,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:02,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:03,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:34:03,233 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:34:03,233 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:34:03,235 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:34:03,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:03,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:34:03,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:34:03,951 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:34:04,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:05,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:05,953 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:34:06,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:06,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:07,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:08,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:09,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:10,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:11,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:11,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:12,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:13,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:14,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:15,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:16,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:16,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:17,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:17,493 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:34:18,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:18,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:34:18,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:34:19,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:20,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:21,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:21,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:22,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:23,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:24,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:25,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:26,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:26,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:27,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:28,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:29,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:30,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:31,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:31,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:32,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:33,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:33,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:34:33,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:34:34,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:34,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:34:34,719 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:34:34,720 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:34:34,721 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:34:34,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:34:35,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:35,983 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:34:36,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:36,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:37,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:38,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:39,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:40,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:41,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:42,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:42,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:43,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:44,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:45,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:46,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:47,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:47,497 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:34:47,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:48,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:48,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:34:48,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:34:49,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:50,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:51,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:52,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:53,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:53,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:54,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:55,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:56,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:57,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:58,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:34:58,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:34:59,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:00,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:01,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:02,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:03,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:35:03,386 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:35:03,387 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:35:03,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:03,389 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:35:03,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:03,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:35:03,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:35:04,014 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:35:04,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:05,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:06,016 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:35:06,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:07,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:08,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:09,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:09,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:10,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:11,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:12,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:13,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:14,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:15,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:15,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:16,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:17,501 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:35:18,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:18,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:35:18,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:35:19,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:20,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:21,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:21,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:22,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:23,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:24,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:25,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:26,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:26,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:27,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:28,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:29,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:30,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:31,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:31,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:32,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:33,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:33,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:35:33,899 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:35:34,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:35,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:35,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:35:35,362 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:35:35,362 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:35:35,364 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:35:36,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:36,046 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:35:36,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:37,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:38,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:38,048 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:35:39,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:40,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:41,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:42,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:42,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:43,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:44,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:45,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:46,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:47,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:47,505 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:35:47,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:48,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:48,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:35:48,897 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:35:49,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:50,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:51,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:52,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:53,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:53,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:35:54,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:55,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:56,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:57,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:58,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:59,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:35:59,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:00,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:01,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:02,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:03,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:03,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:36:03,901 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:36:04,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:05,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:05,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:05,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:36:05,065 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:36:05,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:36:05,066 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:36:05,075 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:36:06,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:06,076 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:36:07,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:08,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:09,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:10,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:10,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:11,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:12,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:13,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:14,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:15,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:16,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:16,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:17,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:17,510 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:36:18,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:18,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:36:18,902 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:36:19,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:20,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:21,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:21,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:22,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:23,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:24,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:25,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:26,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:27,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:27,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:28,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:29,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:30,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:31,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:32,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:32,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:33,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:33,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:36:33,582 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:36:33,583 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:36:33,584 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:36:33,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:36:33,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:36:34,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:34,098 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:36:35,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:36,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:36,099 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:36:37,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:38,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:38,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:39,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:40,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:41,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:42,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:43,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:44,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:44,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:45,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:46,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:47,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:47,512 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:36:48,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:48,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:36:48,991 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:36:49,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:50,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:50,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:51,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:52,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:53,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:54,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:55,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:36:55,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:56,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:57,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:58,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:36:59,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:00,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:00,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:01,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:01,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:37:01,548 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:37:01,548 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:37:01,550 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:37:02,123 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:37:02,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:03,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:03,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:37:03,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:37:04,125 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:37:04,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:05,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:05,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:06,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:07,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:08,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:09,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:10,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:11,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:11,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:12,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:13,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:14,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:15,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:16,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:16,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:17,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:17,516 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:37:18,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:18,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:37:18,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:37:19,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:20,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:21,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:22,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:22,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:23,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:24,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:25,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:26,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:27,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:27,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:28,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:29,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:30,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:31,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:32,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:32,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:32,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:37:32,787 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:37:32,787 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:37:32,789 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:37:33,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:37:33,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:33,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:37:33,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:37:34,153 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:37:34,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:35,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:36,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:37,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:38,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:38,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:39,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:40,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:41,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:42,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:43,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:44,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:44,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:45,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:46,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:47,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:47,520 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:37:48,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:48,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:37:48,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:37:49,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:50,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:50,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:51,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:52,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:53,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:54,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:55,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:37:55,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:56,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:57,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:58,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:37:59,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:00,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:00,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:01,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:02,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:03,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:03,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:38:03,410 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:38:03,410 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:38:03,412 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:38:03,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:38:03,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:38:04,180 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:38:04,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:05,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:06,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:06,181 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:38:06,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:07,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:08,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:09,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:10,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:11,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:11,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:12,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:13,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:14,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:15,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:16,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:16,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:17,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:17,524 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:38:18,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:18,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:38:18,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:38:19,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:20,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:21,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:22,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:22,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:23,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:24,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:25,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:26,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:27,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:28,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:28,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:29,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:30,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:31,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:32,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:33,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:33,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:33,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:38:33,839 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:38:33,839 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:38:33,841 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:38:34,210 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:38:34,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:34,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:38:34,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:38:35,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:36,212 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:38:36,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:37,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:38,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:39,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:39,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:40,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:41,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:42,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:43,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:44,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:45,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:45,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:46,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:47,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:47,528 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:38:48,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:49,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:38:49,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:38:49,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:50,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:50,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:51,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:52,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:53,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:54,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:55,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:56,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:38:56,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:57,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:58,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:38:59,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:00,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:01,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:02,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:02,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:03,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:04,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:39:04,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:39:04,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:04,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:39:04,497 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:39:04,497 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:39:04,499 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:39:05,239 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:39:05,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:06,240 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:39:06,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:07,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:08,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:08,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:09,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:10,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:11,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:12,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:13,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:14,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:14,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:15,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:16,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:17,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:17,532 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:39:18,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:19,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:39:19,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:39:19,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:19,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:20,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:21,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:22,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:23,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:24,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:25,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:25,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:26,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:27,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:28,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:29,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:30,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:30,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:31,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:32,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:33,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:39:33,095 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:39:33,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:39:33,097 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:39:33,270 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:39:33,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:34,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:39:34,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:39:34,271 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:39:34,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:35,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:35,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:36,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:37,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:38,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:39,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:40,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:41,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:41,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:42,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:43,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:44,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:45,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:46,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:46,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:47,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:47,536 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:39:48,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:49,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:39:49,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:39:49,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:50,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:51,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:51,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:52,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:53,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:54,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:55,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:56,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:39:56,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:57,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:58,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:39:59,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:00,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:01,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:02,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:40:02,177 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:40:02,178 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:40:02,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:02,180 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:40:02,303 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:40:02,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:03,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:04,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:40:04,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:40:04,305 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:40:04,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:05,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:06,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:07,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:07,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:08,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:09,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:10,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:11,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:12,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:13,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:13,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:14,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:15,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:16,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:17,541 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:40:17,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:18,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:18,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:19,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:40:19,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:40:19,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:20,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:21,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:22,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:23,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:24,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:24,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:25,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:26,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:27,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:28,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:29,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:29,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:30,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:31,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:32,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:40:32,522 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:40:32,523 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:40:32,524 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:40:32,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:33,334 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:40:33,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:34,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:40:34,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:40:34,335 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:40:34,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:34,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:35,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:36,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:37,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:38,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:39,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:40,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:40,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:41,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:42,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:43,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:44,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:45,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:46,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:46,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:47,545 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:40:47,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:48,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:49,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:40:49,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:40:49,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:50,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:51,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:51,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:52,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:53,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:54,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:55,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:56,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:40:56,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:40:58,352 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:40:59,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:01,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:06,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:06,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:07,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:41:07,819 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:41:07,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:12,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:15,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:16,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:17,549 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:41:17,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:22,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:24,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:25,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:26,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:41:26,359 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:41:27,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:28,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:33,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:34,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:35,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:38,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:43,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:43,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:41:43,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:41:43,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:44,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:47,554 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:41:49,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:54,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:41:56,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:56,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:57,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:41:57,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:41:57,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:58,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:58,676 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:41:59,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:41:59,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:00,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:00,678 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:42:01,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:02,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:03,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:04,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:05,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:05,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:06,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:07,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:08,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:09,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:10,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:11,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:11,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:12,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:42:12,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:42:12,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:13,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:14,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:15,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:16,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:16,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:17,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:17,557 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:42:18,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:19,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:20,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:21,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:22,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:22,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:23,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:24,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:25,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:26,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:27,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:42:27,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:42:27,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:28,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:28,534 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:29,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:30,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:31,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:32,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:33,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:33,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:34,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:35,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:36,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:37,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:38,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:39,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:39,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:40,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:41,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:42,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:42:42,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:42:42,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:43,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:44,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:45,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:45,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:46,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:47,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:47,561 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:42:48,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:42:51,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:42:56,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:01,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:06,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:09,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:09,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:43:09,730 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:43:10,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:11,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:11,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:12,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:13,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:14,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:15,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:16,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:17,565 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:43:17,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:17,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:18,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:19,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:20,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:21,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:22,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:22,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:23,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:24,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:24,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:43:24,728 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:43:25,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:26,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:27,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:27,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:28,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:29,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:30,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:31,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:32,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:33,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:33,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:34,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:35,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:36,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:37,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:38,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:38,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:39,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:39,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:43:39,728 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:43:40,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:41,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:42,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:43,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:43,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:44,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:45,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:46,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:47,569 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:43:47,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:48,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:49,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:49,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:50,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:51,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:52,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:53,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:54,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:54,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:43:54,728 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:43:54,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:43:55,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:56,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:57,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:58,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:43:59,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:00,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:00,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:01,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:02,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:03,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:04,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:05,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:06,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:06,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:07,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:08,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:09,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:44:09,729 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:44:09,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:10,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:11,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:11,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:12,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:13,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:14,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:15,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:16,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:17,573 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:44:17,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:17,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:18,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:19,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:20,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:21,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:22,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:22,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:23,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:24,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:44:24,729 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:44:24,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:25,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:26,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:27,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:27,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:28,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:29,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:30,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:31,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:32,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:33,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:33,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:34,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:35,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:36,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:37,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:38,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:38,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:39,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:44:39,729 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:44:39,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:40,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:41,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:42,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:43,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:43,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:44,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:45,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:46,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:47,578 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:44:47,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:48,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:44:49,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:54,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:44:59,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:04,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:09,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:14,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:17,582 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:45:20,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:25,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:30,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:30,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:45:30,333 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:45:31,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:31,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:32,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:33,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:34,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:34,600 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:45:35,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:36,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:36,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:36,602 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:45:37,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:38,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:39,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:40,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:41,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:42,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:42,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:43,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:44,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:45,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:45:45,331 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:45:45,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:46,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:47,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:47,586 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:45:47,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:48,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:49,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:50,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:51,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:52,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:53,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:53,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:54,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:55,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:56,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:57,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:58,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:45:58,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:45:59,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:00,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:46:00,337 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:46:00,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:01,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:02,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:46:02,403 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:46:02,403 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:46:02,405 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:46:02,633 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:46:02,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:03,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:03,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:04,635 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:46:04,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:05,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:06,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:07,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:08,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:09,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:09,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:10,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:11,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:12,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:13,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:14,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:15,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:15,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:46:15,333 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:46:15,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:16,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:17,590 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:46:17,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:18,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:19,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:20,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:21,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:21,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:22,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:23,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:24,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:25,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:26,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:27,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:28,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:29,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:30,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:30,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:46:30,343 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:46:31,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:31,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:32,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:32,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:46:32,792 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:46:32,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:46:32,793 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:46:33,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:33,675 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:46:34,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:34,676 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:46:35,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:36,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:37,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:37,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:38,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:39,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:40,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:41,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:42,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:43,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:43,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:44,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:45,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:45,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:46:45,340 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:46:46,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:47,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:47,594 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:46:48,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:48,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:49,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:50,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:51,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:52,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:53,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:54,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:54,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:46:55,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:56,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:57,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:58,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:59,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:46:59,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:00,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:00,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:47:00,346 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:47:01,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:02,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:02,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:47:02,428 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:47:02,429 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:47:02,430 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:47:02,716 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:47:03,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:04,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:04,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:04,718 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:47:05,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:06,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:07,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:08,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:09,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:10,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:10,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:11,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:12,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:13,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:14,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:15,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:15,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:15,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:47:15,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:47:16,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:17,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:17,599 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:47:18,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:19,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:20,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:20,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:21,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:22,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:23,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:24,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:25,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:25,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:26,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:27,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:28,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:29,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:30,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:30,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:47:30,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:30,393 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:47:31,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:32,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:33,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:34,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:34,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:47:34,443 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:47:34,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:47:34,445 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:47:34,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:47:35,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:35,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:36,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:36,750 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:47:37,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:38,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:39,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:40,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:41,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:41,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:42,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:43,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:44,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:45,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:45,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:47:45,391 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:47:46,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:46,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:47,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:47,601 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:47:48,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:49,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:50,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:51,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:52,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:52,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:53,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:54,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:55,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:56,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:57,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:57,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:47:58,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:47:59,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:00,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:00,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:48:00,390 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:48:00,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:48:00,808 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:48:00,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:48:00,810 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:48:01,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:01,777 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:48:02,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:02,778 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:48:02,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:03,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:04,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:05,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:06,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:07,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:08,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:08,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:09,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:10,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:11,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:12,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:13,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:14,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:14,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:15,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:15,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:48:15,391 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:48:16,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:17,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:17,605 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:48:18,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:19,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:19,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:20,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:21,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:22,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:23,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:24,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:25,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:25,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:26,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:27,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:28,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:29,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:30,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:30,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:48:30,394 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:48:30,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:31,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:31,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:48:31,316 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:48:31,317 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:48:31,318 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:48:31,809 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:48:32,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:32,810 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:48:33,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:34,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:35,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:36,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:36,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:37,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:38,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:39,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:40,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:41,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:42,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:42,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:43,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:44,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:45,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:45,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:48:45,394 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:48:46,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:47,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:47,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:47,609 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:48:48,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:49,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:50,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:51,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:52,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:53,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:53,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:48:54,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:55,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:56,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:57,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:58,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:59,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:48:59,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:00,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:00,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:49:00,403 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:49:01,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:02,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:03,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:04,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:04,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:49:04,481 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:49:04,482 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:49:04,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:04,483 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:49:04,848 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:49:05,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:06,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:06,849 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:49:07,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:08,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:09,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:10,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:10,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:11,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:12,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:13,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:14,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:15,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:15,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:49:15,399 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:49:15,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:16,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:17,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:17,613 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:49:18,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:19,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:20,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:21,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:21,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:22,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:23,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:24,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:25,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:26,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:27,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:27,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:28,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:29,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:30,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:30,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:49:30,409 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:49:31,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:32,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:32,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:33,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:34,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:35,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:36,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:49:36,026 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:49:36,026 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:49:36,028 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:49:36,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:36,889 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:49:36,889 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:49:37,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:38,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:38,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:39,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:40,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:41,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:42,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:43,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:43,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:44,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:45,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:49:45,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:49:45,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:46,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:47,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:47,617 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:49:48,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:48,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:49,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:50,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:51,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:52,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:53,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:54,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:49:54,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:55,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:56,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:57,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:58,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:49:59,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:00,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:50:00,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:50:00,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:00,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:01,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:02,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:03,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:04,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:05,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:06,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:06,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:07,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:50:07,393 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:50:07,394 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:50:07,396 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:50:07,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:07,918 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:50:08,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:08,919 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:50:09,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:10,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:11,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:12,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:12,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:13,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:14,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:15,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:50:15,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:50:15,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:16,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:17,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:17,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:17,622 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:50:18,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:19,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:20,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:21,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:22,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:23,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:23,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:24,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:25,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:26,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:27,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:28,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:28,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:29,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:30,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:50:30,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:50:30,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:31,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:32,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:33,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:33,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:34,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:35,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:36,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:37,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:38,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:38,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:50:38,923 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:50:38,923 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:50:38,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:38,925 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:50:38,954 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:50:39,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:40,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:40,956 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:50:41,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:42,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:43,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:44,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:44,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:45,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:50:45,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:50:45,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:46,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:47,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:47,625 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:50:48,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:49,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:49,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:50,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:51,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:52,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:53,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:54,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:55,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:50:55,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:56,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:57,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:58,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:50:59,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:00,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:51:00,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:51:00,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:00,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:01,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:02,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:03,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:04,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:05,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:06,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:06,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:07,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:08,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:09,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:51:09,032 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:51:09,032 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:51:09,033 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:51:09,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:09,985 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:51:10,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:10,986 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:51:11,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:12,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:12,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:13,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:14,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:15,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:51:15,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:51:15,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:16,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:17,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:17,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:17,629 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:51:18,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:19,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:20,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:21,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:22,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:23,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:23,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:24,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:25,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:26,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:27,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:28,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:28,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:29,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:30,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:51:30,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:51:30,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:31,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:32,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:33,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:33,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:34,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:35,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:36,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:37,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:38,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:39,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:39,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:40,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:41,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:51:41,436 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:51:41,437 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:51:41,438 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:51:41,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:42,015 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:51:42,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:43,016 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:51:43,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:44,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:44,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:45,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:51:45,405 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:51:45,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:46,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:47,633 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:51:47,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:48,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:49,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:49,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:50,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:51,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:52,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:53,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:54,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:55,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:51:55,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:56,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:57,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:58,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:51:59,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:00,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:52:00,406 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:52:00,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:00,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:01,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:02,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:03,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:04,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:05,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:06,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:06,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:07,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:08,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:09,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:10,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:11,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:11,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:12,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:52:12,462 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:52:12,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:52:12,464 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:52:12,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:13,048 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:52:13,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:14,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:15,049 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:52:15,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:52:15,406 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:52:15,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:16,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:16,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:17,637 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:52:17,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:18,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:19,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:20,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:21,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:22,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:22,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:23,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:24,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:25,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:26,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:27,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:27,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:28,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:29,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:30,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:52:30,406 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:52:30,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:31,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:32,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:32,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:33,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:34,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:35,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:36,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:37,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:38,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:38,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:39,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:40,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:41,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:42,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:43,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:43,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:44,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:45,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:52:45,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:52:45,301 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:52:45,303 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:52:45,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:52:45,720 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:52:45,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:46,078 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:52:46,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:47,079 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:52:47,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:52:47,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:48,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:48,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:49,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:50,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:51,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:52,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:53,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:54,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:52:54,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:55,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:56,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:58,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:59,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:52:59,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:00,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:00,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:53:00,447 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:53:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:02,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:03,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:04,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:04,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:05,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:06,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:07,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:08,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:09,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:10,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:10,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:11,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:12,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:13,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:14,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:15,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:15,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:53:15,450 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:53:15,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:16,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:16,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:53:16,351 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:53:16,351 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:53:16,353 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:53:17,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:17,107 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:53:17,108 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:53:17,646 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:53:18,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:19,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:20,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:21,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:21,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:22,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:23,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:24,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:25,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:26,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:27,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:27,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:28,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:29,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:30,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:30,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:53:30,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:53:31,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:32,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:32,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:33,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:34,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:35,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:36,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:37,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:38,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:38,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:39,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:40,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:41,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:42,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:43,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:43,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:44,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:45,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:45,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:53:45,454 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:53:46,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:47,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:47,653 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:53:48,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:48,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:53:48,302 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:53:48,302 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:53:48,304 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:53:49,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:49,138 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:53:49,138 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:53:49,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:50,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:51,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:52,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:53,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:54,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:54,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:53:55,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:56,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:57,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:58,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:53:59,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:00,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:00,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:54:00,455 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:54:00,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:01,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:02,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:03,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:04,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:05,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:06,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:06,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:07,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:08,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:09,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:10,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:11,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:12,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:12,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:13,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:14,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:15,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:15,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:54:15,460 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:54:16,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:17,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:17,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:17,655 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:54:17,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:54:17,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:54:17,694 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:54:17,695 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:54:18,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:18,173 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:54:19,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:19,174 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:54:20,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:21,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:22,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:23,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:23,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:24,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:25,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:26,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:27,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:28,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:29,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:29,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:30,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:30,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:54:30,457 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:54:31,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:32,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:33,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:34,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:34,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:35,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:36,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:37,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:38,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:39,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:40,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:40,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:41,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:42,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:43,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:44,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:45,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:45,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:54:45,458 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:54:45,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:46,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:47,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:47,664 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:54:48,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:49,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:50,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:50,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:54:50,659 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:54:50,659 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:54:50,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:50,661 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:54:51,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:51,201 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:54:52,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:53,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:53,203 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:54:54,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:55,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:56,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:56,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:54:57,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:58,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:54:59,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:00,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:00,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:55:00,458 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:55:01,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:01,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:02,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:03,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:04,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:05,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:06,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:07,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:07,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:08,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:09,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:10,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:11,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:12,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:13,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:13,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:14,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:15,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:15,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:55:15,468 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:55:16,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:17,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:17,665 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:55:18,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:18,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:19,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:20,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:20,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:55:20,881 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:55:20,882 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:55:20,883 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:55:21,232 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:55:21,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:22,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:23,234 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:55:23,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:23,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:24,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:25,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:26,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:27,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:28,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:29,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:29,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:30,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:30,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:55:30,463 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:55:31,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:32,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:33,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:34,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:34,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:35,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:36,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:37,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:38,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:39,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:40,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:40,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:41,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:42,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:43,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:44,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:45,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:45,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:55:45,469 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:55:45,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:46,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:47,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:47,665 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:55:48,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:49,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:50,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:51,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:51,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:55:51,340 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:55:51,341 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:55:51,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:51,347 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:55:52,267 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:55:52,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:53,268 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:55:53,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:54,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:55,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:56,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:56,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:55:57,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:58,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:55:59,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:00,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:00,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:56:00,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:56:01,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:01,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:02,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:03,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:04,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:05,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:06,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:07,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:07,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:08,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:09,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:10,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:11,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:12,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:12,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:13,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:14,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:15,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:15,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:56:15,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:56:16,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:17,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:17,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:17,670 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:56:18,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:19,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:20,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:21,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:22,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:23,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:23,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:24,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:24,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:56:24,840 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:56:24,841 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:56:24,846 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:56:25,306 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:56:25,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:26,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:27,308 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:56:27,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:28,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:29,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:29,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:30,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:30,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:56:30,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:56:31,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:32,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:33,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:34,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:34,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:35,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:36,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:37,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:38,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:39,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:40,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:40,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:41,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:42,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:43,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:44,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:45,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:45,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:56:45,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:56:45,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:46,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:47,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:47,674 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:56:48,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:49,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:50,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:51,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:51,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:52,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:53,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:54,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:55,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:56,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:56:56,098 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:56:56,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:56:56,101 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:56:56,339 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:56:57,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:56:57,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:57,340 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:56:58,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:56:59,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:00,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:00,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:57:00,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:57:01,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:02,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:02,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:03,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:04,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:05,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:06,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:07,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:08,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:08,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:09,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:10,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:11,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:12,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:13,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:13,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:14,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:15,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:15,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:57:15,466 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:57:16,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:17,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:17,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:57:18,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:18,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:19,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:20,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:21,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:22,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:23,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:24,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:24,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:25,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:26,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:27,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:27,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:57:27,219 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:57:27,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:57:27,223 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:57:27,373 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:57:28,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:29,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:29,374 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:57:30,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:30,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:30,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:57:30,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:57:31,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:32,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:33,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:34,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:35,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:35,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:36,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:37,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:38,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:39,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:40,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:41,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:41,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:42,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:43,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:44,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:45,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:45,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:57:45,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:57:46,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:46,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:47,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:47,682 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:57:48,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:49,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:50,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:51,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:52,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:52,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:53,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:54,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:55,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:56,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:57,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:57,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:57:58,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:59,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:57:59,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:57:59,592 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:57:59,592 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:57:59,597 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:58:00,427 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:58:00,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:00,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:58:00,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:58:01,428 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:58:01,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:02,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:02,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:03,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:04,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:05,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:06,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:07,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:08,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:08,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:09,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:10,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:11,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:12,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:13,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:14,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:14,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:15,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:15,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:58:15,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:58:16,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:17,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:17,686 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:58:18,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:19,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:19,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:20,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:21,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:22,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:23,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:24,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:25,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:25,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:26,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:27,457 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:28,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:28,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:58:28,879 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:58:28,879 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:58:28,881 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:58:29,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:29,462 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:58:30,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:30,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:58:30,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:58:30,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:31,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:31,464 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:58:32,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:33,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:34,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:35,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:36,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:36,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:37,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:38,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:39,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:40,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:41,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:42,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:42,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:43,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:44,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:45,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:45,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:58:45,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:58:46,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:47,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:47,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:47,690 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:58:48,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:49,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:50,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:51,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:52,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:53,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:53,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:54,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:55,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:56,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:57,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:58,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:58:58,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:58:59,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:00,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:59:00,388 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:59:00,389 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:59:00,391 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:59:00,494 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:59:00,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:00,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:59:00,798 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:59:01,495 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:59:01,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:02,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:03,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:03,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:04,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:05,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:06,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:07,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:08,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:09,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:09,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:10,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:11,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:12,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:13,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:14,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:15,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:15,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:59:15,793 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:59:15,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:16,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:17,693 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:59:17,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:18,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:19,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:20,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:20,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:21,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:22,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:23,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:24,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:25,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:26,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:26,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:27,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:28,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:29,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:30,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:59:30,799 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:59:30,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:31,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:31,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:32,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 16:59:32,389 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 16:59:32,390 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 16:59:32,395 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 16:59:32,526 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 16:59:32,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:33,527 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 16:59:33,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:34,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:35,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:36,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:37,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:37,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:38,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:39,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:40,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:41,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:42,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:43,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:43,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:44,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:45,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 16:59:45,797 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 16:59:45,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:46,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:47,697 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 16:59:47,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:48,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:48,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:49,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:50,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:51,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:52,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:53,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:54,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:54,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:55,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:56,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:57,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:58,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 16:59:59,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 16:59:59,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:00,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:00:00,801 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:00:00,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:01,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:02,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:00:02,481 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:00:02,482 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:00:02,487 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:00:02,565 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:00:02,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:03,566 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:00:03,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:04,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:05,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:05,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:06,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:07,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:08,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:09,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:10,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:10,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:11,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:12,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:13,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:14,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:15,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:15,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:00:15,798 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:00:15,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:16,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:17,701 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:00:17,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:18,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:19,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:20,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:20,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:21,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:22,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:23,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:24,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:25,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:26,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:26,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:27,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:28,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:29,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:30,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:00:30,801 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:00:30,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:31,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:31,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:32,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:00:32,135 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:00:32,136 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:00:32,138 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:00:32,594 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:00:32,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:33,595 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:00:33,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:34,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:35,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:36,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:37,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:37,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:38,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:39,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:40,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:41,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:42,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:43,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:43,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:44,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:45,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:00:45,825 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:00:45,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:46,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:47,705 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:00:47,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:48,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:48,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:49,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:50,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:51,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:52,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:53,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:54,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:54,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:55,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:56,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:57,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:58,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:00:59,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:00:59,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:00,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:01:00,828 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:01:00,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:01,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:02,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:03,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:03,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:01:03,939 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:01:03,940 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:01:03,945 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:01:04,627 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:01:04,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:04,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:05,628 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:01:05,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:06,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:07,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:08,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:09,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:10,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:10,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:11,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:12,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:13,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:14,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:15,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:01:15,828 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:01:15,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:15,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:16,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:17,709 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:01:17,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:18,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:19,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:20,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:21,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:21,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:22,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:23,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:24,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:25,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:26,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:26,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:27,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:28,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:29,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:30,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:01:30,829 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:01:30,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:31,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:31,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:32,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:33,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:34,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:01:34,502 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:01:34,502 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:01:34,507 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:01:34,667 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:01:34,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:35,669 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:01:35,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:36,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:37,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:37,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:38,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:39,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:40,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:41,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:42,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:43,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:43,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:44,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:45,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:01:45,829 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:01:45,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:46,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:47,713 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:01:47,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:48,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:48,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:49,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:50,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:51,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:52,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:53,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:54,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:54,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:55,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:56,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:57,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:58,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:01:59,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:01:59,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:00,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:02:00,833 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:02:00,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:01,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:02,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:03,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:02:03,682 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:02:03,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:02:03,685 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:02:03,705 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:02:04,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:04,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:05,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:05,707 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:02:06,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:07,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:08,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:09,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:10,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:10,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:11,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:12,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:13,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:14,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:15,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:15,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:15,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:02:15,831 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:02:16,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:17,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:17,718 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:02:18,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:19,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:20,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:20,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:21,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:22,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:23,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:24,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:25,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:26,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:26,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:27,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:28,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:29,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:30,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:30,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:02:30,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:02:31,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:31,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:32,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:33,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:34,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:35,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:36,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:36,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:02:36,745 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:02:36,745 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:02:36,749 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:02:37,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:37,746 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:02:37,747 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:02:37,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:38,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:39,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:40,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:41,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:42,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:43,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:43,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:44,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:45,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:45,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:02:45,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:02:46,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:47,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:47,722 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:02:48,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:48,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:49,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:50,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:51,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:52,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:53,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:54,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:54,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:02:55,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:56,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:57,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:58,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:59,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:02:59,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:00,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:00,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:03:00,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:03:01,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:02,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:03,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:04,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:04,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:05,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:06,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:07,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:08,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:09,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:09,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:03:09,823 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:03:09,823 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:03:09,828 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:03:10,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:10,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:10,788 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:03:11,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:12,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:12,790 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:03:13,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:14,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:15,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:15,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:03:15,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:03:15,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:16,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:17,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:17,726 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:03:18,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:19,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:20,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:21,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:21,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:22,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:23,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:24,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:25,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:26,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:26,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:27,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:28,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:29,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:30,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:30,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:03:30,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:03:31,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:31,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:32,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:33,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:34,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:35,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:36,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:37,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:37,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:38,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:39,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:40,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:40,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:03:40,691 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:03:40,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:03:40,696 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:03:40,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:03:41,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:42,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:42,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:42,822 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:03:43,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:44,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:45,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:45,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:03:45,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:03:46,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:47,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:47,731 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:03:47,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:48,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:49,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:50,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:51,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:52,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:53,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:53,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:54,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:55,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:56,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:57,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:58,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:03:58,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:03:59,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:00,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:00,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:04:00,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:04:01,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:02,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:03,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:03,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:04,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:05,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:06,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:07,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:08,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:09,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:09,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:10,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:11,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:11,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:04:11,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:04:11,643 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:04:11,648 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:04:11,858 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:04:12,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:13,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:14,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:14,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:14,860 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:04:15,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:15,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:04:15,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:04:16,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:17,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:17,734 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:04:18,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:19,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:19,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:20,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:21,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:22,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:23,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:24,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:25,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:25,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:26,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:27,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:28,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:29,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:30,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:30,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:30,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:04:30,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:04:31,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:32,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:33,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:34,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:35,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:35,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:36,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:37,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:38,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:39,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:39,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:04:39,407 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:04:39,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:04:39,409 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:04:39,891 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:04:40,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:40,893 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:04:41,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:41,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:42,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:43,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:44,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:45,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:45,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:04:45,834 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:04:46,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:46,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:47,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:47,739 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:04:48,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:49,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:50,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:51,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:52,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:52,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:53,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:54,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:55,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:56,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:57,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:57,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:04:58,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:04:59,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:00,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:00,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:05:00,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:05:01,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:02,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:02,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:03,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:04,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:05,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:06,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:07,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:08,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:08,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:09,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:05:09,183 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:05:09,184 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:05:09,186 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:05:09,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:09,927 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:05:10,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:10,929 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:05:11,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:12,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:13,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:14,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:14,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:15,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:15,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:05:15,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:05:16,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:17,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:17,743 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:05:18,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:19,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:19,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:20,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:21,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:22,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:23,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:24,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:25,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:25,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:26,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:27,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:28,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:29,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:30,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:30,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:30,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:05:30,846 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:05:31,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:32,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:33,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:34,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:35,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:35,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:36,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:36,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:05:36,828 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:05:36,829 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:05:36,830 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:05:36,958 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:05:37,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:38,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:38,960 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:05:39,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:40,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:41,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:41,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:42,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:43,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:44,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:45,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:45,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:05:45,841 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:05:46,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:46,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:47,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:47,747 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:05:48,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:49,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:50,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:51,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:52,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:52,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:53,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:54,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:55,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:56,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:57,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:58,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:05:58,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:05:59,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:00,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:00,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:06:00,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:06:01,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:02,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:03,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:03,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:04,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:05,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:06,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:07,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:07,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:06:07,722 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:06:07,723 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:06:07,724 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:06:07,997 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:06:08,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:08,999 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:06:09,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:09,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:10,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:11,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:12,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:13,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:14,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:15,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:15,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:15,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:06:15,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:06:16,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:17,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:17,752 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:06:18,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:19,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:20,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:21,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:21,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:22,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:23,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:24,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:25,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:26,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:26,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:27,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:28,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:29,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:30,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:30,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:06:30,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:06:31,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:31,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:32,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:33,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:34,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:35,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:36,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:37,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:37,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:38,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:39,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:40,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:06:40,435 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:06:40,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:06:40,438 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:06:40,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:41,029 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:06:41,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:42,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:42,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:43,031 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:06:43,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:44,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:45,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:45,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:06:45,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:06:46,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:47,755 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:06:47,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:48,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:48,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:49,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:50,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:51,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:52,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:53,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:53,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:54,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:55,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:56,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:57,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:58,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:06:58,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:06:59,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:00,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:07:00,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:07:00,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:01,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:02,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:03,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:03,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:04,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:05,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:06,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:07,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:08,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:09,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:09,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:10,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:11,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:12,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:07:12,096 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:07:12,097 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:07:12,099 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:07:12,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:13,068 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:07:13,069 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:07:13,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:14,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:15,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:15,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:07:15,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:07:15,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:16,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:17,759 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:07:17,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:18,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:19,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:20,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:20,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:21,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:22,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:23,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:24,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:25,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:25,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:26,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:27,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:28,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:29,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:30,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:07:30,842 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:07:30,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:30,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:31,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:32,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:33,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:34,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:35,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:36,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:36,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:37,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:38,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:39,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:07:39,378 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:07:39,378 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:07:39,380 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:07:39,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:40,096 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:07:40,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:41,097 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:07:41,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:42,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:42,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:43,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:44,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:45,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:07:45,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:07:45,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:46,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:47,763 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:07:47,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:48,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:48,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:49,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:50,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:51,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:52,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:53,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:53,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:54,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:55,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:56,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:57,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:58,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:07:59,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:07:59,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:00,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:08:00,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:08:00,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:01,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:02,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:03,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:04,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:04,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:05,866 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:06,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:07,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:08,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:09,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:10,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:10,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:11,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:12,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:08:12,073 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:08:12,074 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:08:12,076 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:08:12,140 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:08:12,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:13,141 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:08:13,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:14,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:15,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:08:15,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:08:15,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:15,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:16,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:17,767 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:08:17,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:18,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:19,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:20,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:21,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:21,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:22,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:23,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:24,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:25,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:26,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:26,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:27,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:28,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:29,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:30,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:08:30,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:08:30,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:31,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:31,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:32,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:33,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:34,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:35,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:36,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:37,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:37,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:39,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:40,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:41,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:42,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:42,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:43,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:44,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:45,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:45,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:08:45,012 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:08:45,013 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:08:45,014 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:08:45,175 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:08:45,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:08:45,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:08:46,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:47,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:47,177 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:08:47,772 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:08:48,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:48,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:49,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:50,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:51,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:52,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:53,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:53,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:54,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:55,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:56,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:57,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:58,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:08:58,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:08:59,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:00,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:00,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:09:00,843 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:09:01,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:02,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:03,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:03,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:04,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:05,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:06,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:07,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:08,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:09,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:09,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:10,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:11,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:12,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:13,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:14,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:14,882 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:15,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:15,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:09:15,846 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:09:16,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:16,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:09:16,270 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:09:16,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:09:16,273 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:09:17,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:17,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:09:17,203 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:09:17,774 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:09:18,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:19,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:20,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:20,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:21,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:22,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:23,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:24,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:25,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:25,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:26,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:27,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:28,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:29,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:30,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:30,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:09:30,847 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:09:31,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:31,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:32,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:33,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:34,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:35,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:36,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:36,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:37,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:38,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:39,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:40,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:41,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:41,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:42,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:43,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:43,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:09:43,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:09:43,679 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:09:43,681 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:09:44,233 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:09:44,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:45,234 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:09:45,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:45,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:09:45,847 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:09:46,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:46,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:47,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:47,777 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:09:48,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:49,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:50,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:51,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:52,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:52,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:53,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:54,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:55,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:56,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:57,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:57,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:09:58,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:09:59,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:00,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:00,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:10:00,847 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:10:01,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:02,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:02,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:03,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:04,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:05,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:06,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:07,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:08,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:08,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:09,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:10,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:11,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:12,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:13,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:13,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:14,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:15,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:15,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:10:15,613 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:10:15,613 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:10:15,615 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:10:16,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:10:16,027 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:10:16,265 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:10:16,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:17,267 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:10:17,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:17,781 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:10:18,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:19,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:19,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:20,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:21,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:22,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:23,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:24,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:25,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:25,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:26,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:27,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:28,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:29,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:30,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:31,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:10:31,024 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:10:31,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:31,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:32,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:33,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:34,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:35,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:36,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:37,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:37,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:38,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:39,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:40,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:41,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:42,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:42,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:43,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:44,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:45,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:46,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:10:46,024 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:10:46,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:46,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:10:46,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:10:46,394 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:10:46,396 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:10:47,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:47,306 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:10:47,307 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:10:47,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:47,785 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:10:48,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:49,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:50,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:51,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:52,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:53,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:53,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:54,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:55,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:56,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:57,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:58,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:10:59,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:10:59,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:00,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:01,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:11:01,024 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:11:01,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:02,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:03,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:04,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:04,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:05,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:06,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:07,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:08,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:09,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:10,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:10,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:11,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:12,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:13,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:14,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:15,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:15,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:16,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:11:16,031 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:11:16,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:17,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:11:17,235 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:11:17,235 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:11:17,237 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:11:17,333 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:11:17,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:17,789 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:11:18,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:19,334 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:11:19,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:20,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:20,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:21,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:22,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:23,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:24,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:25,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:26,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:26,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:27,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:28,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:29,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:30,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:31,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:11:31,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:11:31,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:31,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:32,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:33,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:34,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:35,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:36,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:37,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:37,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:38,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:39,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:40,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:41,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:42,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:42,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:43,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:44,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:45,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:45,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:11:45,826 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:11:45,826 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:11:45,828 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:11:46,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:11:46,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:11:46,360 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:11:46,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:47,362 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:11:47,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:47,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:47,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:11:48,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:49,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:50,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:51,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:52,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:53,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:53,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:54,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:55,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:56,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:57,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:58,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:11:59,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:11:59,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:00,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:01,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:12:01,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:12:01,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:02,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:03,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:04,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:04,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:05,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:06,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:07,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:08,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:09,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:10,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:10,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:11,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:12,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:13,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:14,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:15,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:15,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:15,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:12:15,788 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:12:15,788 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:12:15,789 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:12:16,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:12:16,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:12:16,394 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:12:16,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:17,395 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:12:17,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:17,797 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:12:18,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:19,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:20,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:20,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:21,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:22,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:23,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:24,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:25,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:26,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:26,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:27,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:28,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:29,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:30,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:31,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:12:31,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:12:31,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:31,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:32,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:33,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:34,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:35,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:36,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:37,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:37,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:38,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:39,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:40,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:41,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:42,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:42,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:43,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:44,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:12:44,253 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:12:44,254 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:12:44,255 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:12:44,426 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:12:44,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:45,427 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:12:45,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:46,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:12:46,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:12:46,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:47,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:47,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:47,801 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:12:48,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:49,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:50,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:51,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:52,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:53,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:53,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:54,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:56,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:56,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:57,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:58,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:12:58,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:12:59,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:00,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:01,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:13:01,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:13:01,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:02,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:03,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:03,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:04,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:05,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:06,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:07,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:08,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:09,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:09,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:10,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:11,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:12,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:13,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:13:13,238 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:13:13,239 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:13:13,240 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:13:13,456 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:13:13,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:14,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:14,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:15,457 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:13:15,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:16,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:13:16,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:13:16,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:17,805 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:13:17,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:18,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:19,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:19,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:20,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:21,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:22,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:23,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:24,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:25,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:25,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:26,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:27,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:28,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:29,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:30,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:30,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:31,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:13:31,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:13:31,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:32,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:33,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:34,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:35,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:35,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:36,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:37,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:38,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:39,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:40,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:41,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:41,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:42,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:43,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:13:43,307 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:13:43,308 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:13:43,309 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:13:43,490 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:13:43,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:44,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:45,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:13:45,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:46,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:13:46,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:13:46,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:46,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:47,809 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:13:47,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:48,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:49,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:50,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:51,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:52,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:52,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:53,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:54,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:55,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:56,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:57,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:13:57,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:58,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:13:59,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:00,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:01,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:14:01,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:14:01,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:02,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:02,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:03,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:04,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:05,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:07,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:07,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:08,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:08,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:09,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:10,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:11,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:12,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:13,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:13,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:14,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:14:14,685 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:14:14,686 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:14:14,687 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:14:15,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:15,526 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:14:16,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:16,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:14:16,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:14:17,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:17,528 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:14:17,814 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:14:18,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:18,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:19,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:20,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:21,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:22,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:23,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:24,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:24,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:25,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:26,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:27,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:28,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:29,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:30,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:30,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:31,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:31,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:14:31,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:14:32,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:33,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:34,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:35,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:35,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:36,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:37,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:38,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:39,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:40,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:41,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:41,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:42,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:43,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:44,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:45,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:46,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:46,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:14:46,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:14:46,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:14:46,364 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:14:46,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:46,365 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:14:46,367 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:14:46,557 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:14:47,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:47,558 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:14:47,817 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:14:48,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:49,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:50,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:51,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:52,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:52,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:53,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:54,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:55,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:56,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:57,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:14:57,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:58,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:14:59,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:00,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:01,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:01,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:15:01,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:15:02,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:02,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:03,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:04,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:05,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:06,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:07,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:08,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:08,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:09,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:10,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:11,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:12,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:13,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:13,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:13,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:15:13,489 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:15:13,490 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:15:13,492 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:15:13,582 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:15:14,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:15,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:15,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:15:16,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:16,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:15:16,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:15:17,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:17,821 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:15:18,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:18,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:19,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:20,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:21,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:22,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:23,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:24,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:24,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:25,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:26,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:27,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:28,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:29,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:29,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:30,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:31,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:31,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:15:31,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:15:32,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:33,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:34,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:34,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:35,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:36,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:37,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:38,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:39,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:40,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:40,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:41,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:42,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:43,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:43,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:15:43,443 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:15:43,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:15:43,445 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:15:43,611 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:15:44,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:45,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:45,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:45,612 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:15:46,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:46,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:15:46,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:15:47,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:47,825 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:15:48,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:49,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:50,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:50,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:51,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:52,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:53,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:54,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:55,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:56,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:56,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:15:57,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:58,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:15:59,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:00,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:01,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:01,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:16:01,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:16:01,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:02,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:03,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:04,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:05,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:06,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:07,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:07,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:08,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:09,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:10,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:11,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:12,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:12,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:13,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:13,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:16:13,688 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:16:13,688 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:16:13,689 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:16:14,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:14,647 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:16:15,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:15,648 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:16:16,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:16,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:16:16,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:16:17,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:17,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:17,829 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:16:18,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:19,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:20,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:21,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:22,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:23,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:23,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:24,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:25,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:26,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:27,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:28,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:28,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:29,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:30,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:31,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:31,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:16:31,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:16:32,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:33,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:33,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:34,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:35,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:36,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:37,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:38,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:39,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:39,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:40,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:41,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:42,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:43,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:43,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:16:43,538 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:16:43,538 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:16:43,540 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:16:43,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:16:44,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:44,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:45,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:45,682 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:16:46,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:16:46,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:16:46,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:47,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:47,834 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:16:48,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:49,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:49,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:50,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:51,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:52,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:53,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:54,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:55,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:55,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:16:56,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:57,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:58,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:16:59,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:00,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:00,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:01,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:17:01,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:17:01,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:02,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:03,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:04,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:05,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:05,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:06,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:07,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:08,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:09,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:10,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:11,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:11,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:12,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:13,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:14,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:15,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:15,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:17:15,532 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:17:15,533 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:17:15,534 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:17:15,712 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:17:16,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:17:16,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:17:16,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:16,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:17,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:17,714 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:17:17,838 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:17:18,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:19,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:20,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:21,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:22,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:22,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:23,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:24,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:25,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:26,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:27,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:28,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:28,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:29,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:30,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:31,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:17:31,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:17:31,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:32,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:33,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:33,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:34,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:35,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:36,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:37,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:38,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:39,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:39,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:40,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:41,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:42,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:43,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:44,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:45,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:45,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:45,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:17:45,455 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:17:45,455 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:17:45,457 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:17:45,743 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:17:46,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:17:46,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:17:46,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:47,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:47,745 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:17:47,842 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:17:48,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:49,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:50,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:50,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:51,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:52,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:53,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:54,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:55,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:56,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:56,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:17:57,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:58,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:17:59,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:00,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:01,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:18:01,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:18:01,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:01,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:02,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:03,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:04,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:05,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:06,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:07,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:07,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:08,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:09,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:10,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:11,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:12,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:12,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:13,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:14,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:15,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:16,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:18:16,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:18:16,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:17,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:17,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:17,845 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:18:18,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:18:18,104 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:18:18,104 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:18:18,106 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:18:18,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:18,776 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:18:19,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:19,777 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:18:20,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:21,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:22,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:23,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:23,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:24,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:25,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:26,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:27,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:28,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:29,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:29,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:30,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:31,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:18:31,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:18:31,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:32,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:33,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:34,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:34,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:35,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:36,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:37,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:38,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:39,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:40,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:40,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:41,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:42,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:43,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:44,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:45,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:45,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:46,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:18:46,251 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:18:46,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:47,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:47,853 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:18:48,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:49,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:49,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:18:49,542 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:18:49,543 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:18:49,544 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:18:49,808 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:18:50,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:50,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:51,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:51,810 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:18:52,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:53,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:54,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:55,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:56,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:18:56,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:57,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:58,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:18:59,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:00,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:01,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:19:01,252 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:19:01,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:01,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:02,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:03,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:04,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:05,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:06,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:07,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:07,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:08,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:09,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:10,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:11,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:12,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:13,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:13,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:14,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:15,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:16,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:19:16,258 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:19:16,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:17,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:17,858 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:19:18,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:18,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:19,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:20,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:19:20,515 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:19:20,515 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:19:20,517 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:19:20,840 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:19:20,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:21,841 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:19:21,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:22,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:23,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:24,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:24,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:25,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:26,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:27,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:28,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:29,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:30,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:30,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:31,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:19:31,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:19:31,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:32,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:34,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:35,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:35,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:36,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:37,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:38,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:39,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:40,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:41,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:41,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:42,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:43,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:44,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:45,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:46,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:46,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:19:46,258 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:19:46,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:47,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:47,866 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:19:48,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:49,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:50,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:50,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:19:50,976 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:19:50,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:19:50,978 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:19:51,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:51,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:51,873 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:19:51,873 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:19:52,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:53,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:54,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:55,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:56,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:57,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:57,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:19:58,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:19:59,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:00,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:01,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:20:01,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:20:01,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:02,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:02,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:03,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:04,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:05,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:06,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:07,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:08,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:08,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:09,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:10,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:11,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:12,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:13,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:14,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:14,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:15,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:16,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:20:16,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:20:16,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:17,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:17,864 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:20:18,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:19,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:19,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:20,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:21,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:22,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:23,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:24,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:25,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:25,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:26,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:26,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:20:26,763 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:20:26,763 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:20:26,768 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:20:26,910 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:20:27,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:27,911 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:20:28,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:29,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:30,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:30,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:31,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:20:31,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:20:31,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:32,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:33,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:34,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:35,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:36,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:36,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:37,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:38,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:39,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:40,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:41,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:41,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:42,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:43,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:44,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:45,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:46,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:20:46,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:20:46,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:47,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:47,868 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:20:47,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:48,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:49,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:50,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:51,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:52,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:52,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:53,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:54,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:55,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:56,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:57,400 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:20:57,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:20:57,463 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:20:57,464 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:20:57,466 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:20:57,942 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:20:58,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:59,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:20:59,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:21:00,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:01,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:21:01,257 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:21:01,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:02,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:03,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:03,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:04,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:05,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:06,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:07,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:08,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:08,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:09,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:10,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:11,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:12,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:13,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:13,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:14,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:15,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:16,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:21:16,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:21:16,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:17,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:17,876 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:21:18,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:18,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:19,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:20,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:21,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:22,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:23,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:23,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:21:23,436 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:21:23,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:21:23,438 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:21:23,973 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:21:24,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:24,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:25,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:25,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:21:26,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:27,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:28,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:29,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:30,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:30,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:31,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:21:31,257 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:21:31,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:32,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:33,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:34,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:35,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:36,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:36,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:37,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:38,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:39,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:40,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:41,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:42,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:42,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:43,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:44,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:45,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:46,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:21:46,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:21:46,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:47,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:47,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:47,882 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:21:48,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:49,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:50,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:51,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:21:51,020 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:21:51,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:21:51,022 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:21:51,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:52,001 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:21:52,002 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:21:52,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:52,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:53,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:54,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:55,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:56,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:57,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:58,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:21:58,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:21:59,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:00,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:01,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:22:01,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:22:01,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:02,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:03,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:04,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:04,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:05,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:06,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:07,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:08,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:09,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:09,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:10,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:11,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:12,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:13,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:14,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:15,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:15,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:16,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:22:16,259 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:22:16,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:17,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:17,883 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:22:18,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:19,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:20,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:20,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:21,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:22:21,191 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:22:21,192 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:22:21,193 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:22:21,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:22,037 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:22:22,037 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:22:22,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:23,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:24,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:25,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:26,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:26,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:27,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:28,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:29,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:30,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:31,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:22:31,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:22:31,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:32,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:32,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:33,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:34,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:35,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:36,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:37,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:37,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:38,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:39,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:40,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:41,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:42,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:42,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:43,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:44,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:45,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:46,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:22:46,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:22:46,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:47,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:47,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:47,879 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:22:48,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:49,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:50,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:51,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:52,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:22:52,533 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:22:52,533 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:22:52,534 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:22:52,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:53,075 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:22:53,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:53,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:54,076 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:22:54,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:55,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:56,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:57,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:58,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:22:58,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:22:59,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:00,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:01,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:23:01,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:23:01,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:02,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:03,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:04,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:04,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:05,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:06,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:07,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:08,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:09,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:09,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:10,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:11,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:12,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:13,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:14,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:15,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:15,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:16,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:23:16,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:23:16,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:17,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:17,888 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:23:18,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:19,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:20,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:20,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:21,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:22,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:23,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:23,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:23:23,779 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:23:23,780 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:23:23,781 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:23:24,103 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:23:24,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:25,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:26,106 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:23:26,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:26,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:27,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:28,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:29,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:30,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:31,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:23:31,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:23:31,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:32,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:32,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:33,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:34,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:35,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:36,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:37,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:37,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:38,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:39,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:40,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:41,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:42,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:43,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:43,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:44,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:45,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:46,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:23:46,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:23:46,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:47,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:47,889 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:23:48,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:48,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:49,697 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:50,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:51,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:52,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:53,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:23:53,141 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:23:53,141 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:23:53,142 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:23:53,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:54,132 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:23:54,133 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:23:54,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:54,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:55,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:56,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:57,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:58,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:23:59,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:23:59,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:00,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:01,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:24:01,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:24:01,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:02,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:03,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:04,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:05,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:05,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:06,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:07,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:08,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:09,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:10,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:10,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:11,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:12,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:13,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:14,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:15,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:16,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:24:16,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:24:16,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:16,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:17,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:17,890 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:24:18,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:19,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:20,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:21,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:21,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:22,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:23,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:24:23,642 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:24:23,642 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:24:23,644 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:24:24,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:24,165 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:24:25,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:26,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:26,167 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:24:26,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:27,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:28,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:29,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:30,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:31,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:31,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:24:31,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:24:32,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:32,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:33,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:34,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:35,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:36,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:37,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:37,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:38,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:39,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:40,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:41,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:42,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:43,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:43,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:44,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:45,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:46,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:46,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:24:46,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:24:47,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:47,893 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:24:48,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:48,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:49,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:50,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:51,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:52,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:52,881 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:24:52,882 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:24:52,883 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:24:52,888 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:24:53,193 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:24:53,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:53,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:24:54,194 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:24:54,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:55,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:56,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:57,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:58,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:59,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:24:59,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:00,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:01,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:25:01,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:25:01,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:02,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:03,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:04,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:05,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:05,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:06,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:07,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:08,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:09,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:10,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:10,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:11,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:12,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:13,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:14,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:15,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:15,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:16,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:25:16,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:25:16,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:17,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:17,897 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:25:18,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:19,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:20,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:20,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:21,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:22,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:23,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:25:23,066 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:25:23,066 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:25:23,071 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:25:23,228 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:25:23,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:24,229 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:25:24,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:25,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:26,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:26,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:27,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:28,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:29,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:30,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:31,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:25:31,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:25:31,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:31,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:32,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:33,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:34,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:35,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:36,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:36,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:37,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:38,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:39,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:40,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:41,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:41,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:42,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:43,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:44,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:45,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:46,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:25:46,275 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:25:46,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:46,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:47,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:47,910 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:25:48,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:49,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:50,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:51,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:51,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:52,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:25:52,030 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:25:52,030 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:25:52,032 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:25:52,259 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:25:52,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:53,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:54,261 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:25:54,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:55,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:56,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:56,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:25:57,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:58,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:25:59,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:00,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:01,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:26:01,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:26:01,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:01,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:02,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:03,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:04,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:05,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:06,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:07,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:07,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:08,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:09,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:10,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:11,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:12,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:13,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:13,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:14,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:15,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:16,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:26:16,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:26:16,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:17,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:17,901 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:26:18,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:18,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:19,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:20,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:21,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:22,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:22,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:26:22,850 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:26:22,850 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:26:22,852 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:26:23,302 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:26:23,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:24,303 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:26:24,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:24,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:25,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:26,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:27,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:28,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:29,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:30,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:30,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:31,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:26:31,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:26:31,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:32,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:33,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:34,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:35,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:36,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:36,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:37,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:38,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:39,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:40,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:41,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:42,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:42,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:43,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:44,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:45,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:46,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:26:46,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:26:46,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:47,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:47,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:47,912 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:26:48,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:49,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:50,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:51,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:52,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:52,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:53,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:26:53,181 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:26:53,181 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:26:53,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:26:53,334 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:26:53,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:54,335 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:26:54,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:55,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:56,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:57,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:58,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:26:58,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:26:59,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:00,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:01,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:27:01,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:27:01,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:02,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:03,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:04,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:04,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:05,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:06,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:07,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:08,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:09,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:09,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:10,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:11,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:12,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:13,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:14,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:15,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:15,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:16,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:27:16,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:27:16,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:17,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:17,910 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:27:18,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:19,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:20,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:20,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:21,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:22,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:27:22,397 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:27:22,397 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:27:22,399 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:27:22,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:23,362 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:27:23,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:24,363 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:27:24,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:25,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:26,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:26,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:27,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:28,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:29,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:30,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:31,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:27:31,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:27:31,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:31,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:32,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:33,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:34,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:35,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:36,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:36,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:37,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:38,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:39,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:40,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:41,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:41,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:42,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:43,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:44,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:45,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:46,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:27:46,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:27:46,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:47,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:47,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:47,915 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:27:48,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:49,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:50,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:51,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:52,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:52,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:53,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:54,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:27:54,436 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:27:54,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:27:54,440 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:27:54,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:55,391 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:27:55,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:56,392 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:27:56,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:57,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:58,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:27:58,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:27:59,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:00,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:01,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:28:01,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:28:01,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:02,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:03,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:04,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:04,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:05,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:06,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:07,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:08,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:09,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:10,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:10,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:11,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:12,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:13,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:14,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:15,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:15,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:16,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:28:16,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:28:16,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:17,919 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:28:17,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:18,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:19,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:20,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:20,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:21,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:22,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:23,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:24,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:25,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:28:25,065 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:28:25,065 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:28:25,067 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:28:25,424 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:28:25,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:26,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:26,425 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:28:26,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:27,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:28,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:29,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:30,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:31,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:28:31,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:28:31,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:31,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:32,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:33,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:34,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:35,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:36,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:36,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:37,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:39,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:40,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:41,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:41,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:42,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:43,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:44,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:45,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:46,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:46,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:28:46,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:28:47,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:47,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:47,923 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:28:48,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:49,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:50,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:51,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:52,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:52,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:53,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:54,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:55,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:56,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:57,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:57,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:28:58,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:28:58,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:28:58,360 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:28:58,360 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:28:58,365 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:28:58,457 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:28:59,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:00,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:00,458 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:29:01,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:01,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:29:01,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:29:02,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:03,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:03,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:04,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:05,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:06,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:07,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:08,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:08,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:09,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:10,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:11,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:12,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:13,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:13,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:14,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:15,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:16,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:16,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:29:16,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:29:17,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:17,927 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:29:18,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:18,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:19,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:20,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:21,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:22,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:23,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:24,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:24,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:25,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:26,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:27,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:27,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:29:27,738 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:29:27,738 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:29:27,740 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:29:28,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:28,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:29:28,493 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:29:29,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:29,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:30,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:31,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:29:31,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:29:31,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:32,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:33,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:34,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:35,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:35,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:36,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:37,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:38,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:39,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:40,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:40,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:41,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:42,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:43,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:44,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:45,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:45,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:46,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:29:46,265 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:29:46,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:47,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:47,931 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:29:48,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:49,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:50,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:50,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:51,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:52,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:53,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:54,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:55,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:56,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:56,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:29:57,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:58,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:59,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:29:59,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:29:59,906 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:29:59,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:29:59,911 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:30:00,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:00,522 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:30:00,523 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:30:01,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:30:01,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:30:01,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:01,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:02,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:03,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:04,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:05,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:06,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:06,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:07,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:08,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:09,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:10,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:11,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:12,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:12,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:13,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:14,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:15,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:16,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:30:16,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:30:16,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:17,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:17,936 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:30:17,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:18,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:19,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:20,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:21,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:22,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:23,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:23,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:24,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:25,323 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:26,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:27,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:28,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:28,686 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:29,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:30,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:30,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:30:30,680 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:30:30,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:30:30,685 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:30:31,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:30:31,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:30:31,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:31,552 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:30:32,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:32,553 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:30:33,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:34,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:34,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:35,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:36,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:37,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:38,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:39,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:39,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:40,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:41,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:42,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:43,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:44,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:45,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:45,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:46,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:30:46,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:30:46,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:47,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:47,940 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:30:48,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:49,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:50,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:50,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:51,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:52,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:53,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:54,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:55,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:56,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:56,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:30:57,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:58,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:30:59,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:00,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:31:00,087 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:31:00,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:31:00,089 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:31:00,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:00,583 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:31:00,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:31:01,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:31:01,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:31:01,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:02,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:02,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:03,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:04,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:05,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:06,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:07,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:07,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:08,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:09,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:10,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:11,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:12,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:12,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:13,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:14,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:15,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:16,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:31:16,267 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:31:16,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:17,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:17,944 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:31:17,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:18,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:19,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:20,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:21,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:22,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:23,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:23,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:24,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:25,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:26,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:27,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:28,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:28,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:29,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:30,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:30,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:31:30,525 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:31:30,525 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:31:30,527 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:31:30,615 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:31:31,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:31:31,267 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:31:31,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:32,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:32,617 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:31:33,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:34,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:34,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:35,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:36,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:37,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:38,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:39,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:40,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:40,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:41,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:42,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:43,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:44,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:45,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:46,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:31:46,267 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:31:46,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:46,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:47,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:47,948 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:31:48,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:49,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:50,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:51,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:51,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:52,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:53,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:54,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:55,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:56,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:56,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:31:57,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:58,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:31:59,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:00,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:01,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:32:01,277 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:32:01,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:01,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:02,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:03,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:04,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:05,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:05,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:32:05,578 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:32:05,578 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:32:05,580 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:32:05,653 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:32:06,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:06,655 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:32:06,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:07,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:08,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:09,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:10,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:11,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:11,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:12,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:13,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:14,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:15,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:16,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:32:16,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:32:16,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:17,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:17,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:17,951 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:32:18,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:19,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:20,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:21,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:22,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:22,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:23,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:24,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:25,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:26,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:27,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:28,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:28,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:29,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:30,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:31,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:32:31,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:32:31,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:32,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:33,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:34,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:34,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:35,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:36,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:37,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:38,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:39,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:39,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:40,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:41,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:32:41,503 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:32:41,504 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:32:41,505 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:32:41,695 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:32:41,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:42,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:43,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:43,697 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:32:44,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:44,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:45,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:46,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:32:46,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:32:46,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:47,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:47,955 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:32:48,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:49,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:49,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:50,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:51,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:52,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:53,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:54,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:55,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:55,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:32:56,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:57,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:58,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:32:59,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:00,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:00,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:01,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:33:01,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:33:01,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:02,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:03,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:04,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:05,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:05,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:06,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:07,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:08,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:09,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:10,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:33:10,185 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:33:10,185 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:33:10,187 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:33:10,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:10,723 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:33:11,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:11,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:11,724 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:33:12,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:13,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:14,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:15,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:16,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:33:16,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:33:16,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:16,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:17,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:17,959 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:33:18,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:19,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:20,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:21,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:21,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:22,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:23,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:24,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:25,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:26,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:26,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:27,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:28,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:29,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:30,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:31,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:33:31,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:33:31,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:32,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:32,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:33,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:34,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:35,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:36,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:37,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:37,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:38,674 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:39,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:33:39,471 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:33:39,471 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:33:39,472 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:33:39,753 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:33:39,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:40,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:41,755 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:33:41,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:42,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:43,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:43,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:44,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:45,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:46,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:33:46,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:33:46,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:47,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:47,963 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:33:48,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:48,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:49,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:50,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:51,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:52,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:53,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:54,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:54,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:33:55,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:56,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:57,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:58,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:33:59,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:00,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:00,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:01,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:34:01,270 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:34:01,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:02,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:03,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:04,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:05,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:05,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:06,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:07,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:08,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:09,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:10,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:34:10,500 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:34:10,500 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:34:10,501 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:34:10,787 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:34:10,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:11,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:11,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:12,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:13,790 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:34:13,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:14,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:15,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:16,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:34:16,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:34:16,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:16,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:17,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:17,967 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:34:18,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:19,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:20,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:21,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:21,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:22,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:23,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:24,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:25,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:26,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:27,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:27,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:28,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:29,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:30,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:31,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:34:31,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:34:31,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:32,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:33,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:33,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:34,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:35,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:36,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:37,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:38,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:38,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:39,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:40,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:41,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:42,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:34:42,551 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:34:42,551 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:34:42,552 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:34:42,819 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:34:42,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:43,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:44,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:44,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:45,821 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:34:45,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:46,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:34:46,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:34:46,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:47,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:47,971 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:34:48,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:49,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:49,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:50,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:51,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:52,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:53,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:54,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:55,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:34:55,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:56,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:57,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:58,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:34:59,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:00,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:00,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:01,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:35:01,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:35:01,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:02,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:03,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:04,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:05,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:06,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:06,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:07,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:08,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:09,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:10,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:11,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:11,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:12,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:13,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:14,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:15,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:35:15,090 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:35:15,091 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:35:15,093 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:35:15,855 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:35:15,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:16,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:35:16,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:35:16,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:16,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:17,857 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:35:17,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:17,975 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:35:18,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:19,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:20,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:21,938 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:21,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:22,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:23,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:24,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:25,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:26,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:27,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:28,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:29,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:30,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:31,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:31,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:35:31,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:35:32,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:32,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:33,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:34,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:35,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:36,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:37,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:38,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:38,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:39,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:40,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:41,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:42,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:43,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:43,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:44,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:45,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:35:45,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:35:45,169 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:35:45,171 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:35:45,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:45,890 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:35:46,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:35:46,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:35:46,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:47,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:47,892 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:35:47,980 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:35:48,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:48,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:49,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:50,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:51,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:52,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:53,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:54,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:54,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:55,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:56,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:57,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:58,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:35:59,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:35:59,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:00,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:01,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:36:01,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:36:01,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:02,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:03,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:04,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:05,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:05,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:06,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:07,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:08,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:09,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:10,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:11,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:11,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:12,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:13,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:14,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:15,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:16,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:36:16,277 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:36:16,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:16,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:16,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:36:16,857 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:36:16,858 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:36:16,859 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:36:16,931 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:36:17,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:17,983 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:36:18,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:19,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:19,933 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:36:20,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:21,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:21,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:22,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:23,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:24,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:25,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:26,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:27,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:27,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:28,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:29,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:30,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:31,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:36:31,278 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:36:31,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:32,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:32,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:33,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:34,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:35,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:36,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:37,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:37,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:38,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:39,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:40,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:41,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:42,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:43,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:43,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:44,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:45,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:46,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:36:46,286 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:36:46,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:47,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:47,986 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:36:48,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:48,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:36:48,643 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:36:48,643 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:36:48,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:48,645 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:36:48,966 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:36:49,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:50,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:51,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:51,968 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:36:52,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:53,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:54,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:36:54,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:55,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:56,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:57,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:58,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:36:59,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:00,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:00,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:01,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:37:01,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:37:01,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:02,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:03,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:04,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:05,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:05,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:06,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:07,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:08,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:09,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:10,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:11,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:11,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:12,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:13,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:14,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:15,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:16,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:37:16,287 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:37:16,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:16,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:17,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:17,996 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:37:18,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:19,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:19,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:37:19,917 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:37:19,917 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:37:19,919 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:37:20,008 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:37:20,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:21,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:21,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:22,010 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:37:22,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:23,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:24,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:25,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:26,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:27,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:27,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:28,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:29,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:30,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:31,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:37:31,284 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:37:31,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:32,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:33,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:33,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:34,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:35,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:36,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:37,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:38,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:38,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:39,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:40,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:41,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:42,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:43,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:44,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:44,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:45,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:46,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:37:46,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:37:46,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:47,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:47,999 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:37:48,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:49,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:50,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:50,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:51,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:37:51,732 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:37:51,732 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:37:51,738 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:37:51,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:52,038 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:37:52,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:53,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:54,040 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:37:54,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:55,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:37:55,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:56,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:57,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:58,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:37:59,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:00,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:01,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:38:01,288 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:38:01,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:01,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:02,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:03,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:04,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:05,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:06,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:07,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:07,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:08,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:09,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:10,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:11,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:12,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:13,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:13,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:14,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:15,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:16,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:38:16,294 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:38:16,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:17,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:18,004 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:38:18,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:19,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:19,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:20,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:38:20,626 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:38:20,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:38:20,628 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:38:21,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:21,071 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:38:22,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:22,072 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:38:23,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:24,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:24,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:25,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:26,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:27,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:28,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:29,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:30,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:30,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:31,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:31,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:38:31,291 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:38:32,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:33,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:34,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:35,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:35,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:36,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:37,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:38,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:39,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:40,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:41,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:41,296 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:42,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:43,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:44,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:45,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:46,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:46,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:38:46,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:46,302 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:38:47,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:48,007 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:38:48,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:49,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:50,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:51,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:51,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:38:51,211 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:38:51,212 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:38:51,213 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:38:52,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:52,105 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:38:52,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:53,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:54,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:54,107 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:38:55,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:56,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:57,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:57,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:38:58,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:38:59,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:00,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:01,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:39:01,295 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:39:02,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:03,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:03,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:04,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:05,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:06,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:07,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:08,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:08,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:09,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:10,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:11,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:12,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:13,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:14,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:14,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:15,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:16,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:16,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:39:16,303 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:39:17,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:18,007 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:39:18,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:19,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:20,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:20,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:21,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:21,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:39:21,742 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:39:21,742 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:39:21,747 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:39:22,139 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:39:22,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:23,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:24,141 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:39:24,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:25,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:25,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:26,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:27,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:28,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:29,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:30,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:31,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:31,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:39:31,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:39:31,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:32,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:33,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:34,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:35,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:36,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:37,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:37,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:38,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:39,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:40,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:41,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:42,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:43,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:43,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:44,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:45,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:46,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:46,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:39:46,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:39:47,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:48,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:39:48,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:49,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:49,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:50,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:51,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:52,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:52,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:39:52,864 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:39:52,865 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:39:52,868 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:39:53,168 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:39:53,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:54,169 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:39:54,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:54,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:39:55,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:56,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:57,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:58,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:39:59,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:00,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:00,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:01,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:01,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:40:01,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:40:02,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:03,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:04,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:05,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:06,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:06,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:07,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:08,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:09,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:10,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:11,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:11,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:12,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:13,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:14,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:15,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:16,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:16,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:40:16,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:40:16,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:17,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:18,016 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:40:18,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:19,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:20,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:21,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:22,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:22,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:23,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:24,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:24,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:40:24,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:40:24,383 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:40:24,389 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:40:25,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:40:25,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:26,203 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:40:26,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:27,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:27,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:28,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:29,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:30,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:31,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:31,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:40:31,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:40:32,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:33,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:33,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:34,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:35,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:36,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:37,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:38,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:38,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:39,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:40,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:41,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:42,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:43,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:44,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:44,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:45,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:46,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:46,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:40:46,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:40:47,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:48,020 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:40:48,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:49,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:50,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:50,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:51,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:52,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:53,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:54,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:54,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:40:54,848 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:40:54,848 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:40:54,851 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:40:55,235 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:40:55,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:55,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:40:56,237 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:40:56,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:57,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:58,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:40:59,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:00,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:01,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:01,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:41:01,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:41:01,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:02,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:03,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:04,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:05,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:06,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:07,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:07,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:08,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:09,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:10,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:11,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:12,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:13,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:13,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:14,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:15,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:16,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:16,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:41:16,298 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:41:17,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:18,024 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:41:18,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:19,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:19,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:20,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:21,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:22,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:23,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:24,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:41:24,302 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:41:24,303 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:41:24,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:24,305 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:41:24,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:25,269 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:41:25,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:26,270 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:41:26,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:27,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:28,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:29,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:29,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:30,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:31,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:41:31,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:41:31,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:32,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:33,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:34,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:35,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:35,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:36,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:37,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:38,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:39,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:40,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:40,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:41,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:42,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:43,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:44,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:45,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:45,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:46,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:41:46,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:41:46,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:47,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:48,036 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:41:48,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:49,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:50,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:51,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:51,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:52,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:53,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:53,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:41:53,906 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:41:53,907 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:41:53,909 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:41:54,307 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:41:54,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:55,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:56,310 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:41:56,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:56,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:41:57,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:58,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:41:59,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:00,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:01,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:42:01,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:42:01,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:02,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:02,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:03,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:04,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:05,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:06,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:07,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:07,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:08,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:09,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:10,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:11,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:12,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:12,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:13,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:14,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:15,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:16,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:42:16,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:42:16,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:17,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:17,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:18,033 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:42:18,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:19,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:20,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:21,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:22,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:23,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:23,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:24,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:25,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:26,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:26,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:42:26,406 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:42:26,407 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:42:26,412 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:42:27,343 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:42:27,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:28,344 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:42:28,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:28,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:29,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:30,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:31,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:42:31,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:42:31,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:32,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:33,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:34,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:34,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:35,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:36,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:37,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:38,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:39,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:39,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:40,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:41,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:42,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:43,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:44,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:45,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:45,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:46,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:42:46,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:42:46,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:47,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:48,037 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:42:48,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:49,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:50,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:51,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:51,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:52,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:53,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:54,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:55,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:56,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:42:56,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:57,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:42:57,345 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:42:57,345 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:42:57,347 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:42:57,378 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:42:58,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:42:58,380 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:42:59,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:00,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:01,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:43:01,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:43:01,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:02,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:02,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:03,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:04,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:05,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:06,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:07,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:07,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:08,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:09,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:10,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:11,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:12,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:12,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:13,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:14,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:15,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:16,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:43:16,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:43:16,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:17,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:18,041 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:43:18,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:18,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:19,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:20,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:21,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:22,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:23,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:23,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:24,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:25,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:26,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:27,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:27,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:43:27,653 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:43:27,654 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:43:27,657 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:43:28,411 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:43:28,411 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:43:28,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:28,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:29,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:30,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:31,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:43:31,299 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:43:31,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:32,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:33,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:34,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:34,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:35,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:36,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:37,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:38,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:39,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:39,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:40,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:41,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:42,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:43,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:44,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:44,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:45,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:46,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:43:46,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:43:46,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:47,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:48,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:43:48,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:49,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:50,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:50,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:51,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:52,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:53,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:54,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:55,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:55,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:43:56,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:57,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:57,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:43:57,471 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:43:57,471 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:43:57,473 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:43:58,445 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:43:58,445 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:43:58,553 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:43:59,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:00,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:00,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:01,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:44:01,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:44:01,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:02,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:03,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:04,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:05,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:05,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:06,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:07,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:08,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:09,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:10,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:11,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:11,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:12,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:13,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:14,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:15,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:16,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:44:16,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:44:16,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:17,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:17,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:18,048 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:44:18,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:19,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:20,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:21,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:22,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:22,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:23,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:24,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:25,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:26,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:27,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:27,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:28,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:29,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:44:29,238 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:44:29,238 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:44:29,241 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:44:29,491 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:44:29,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:30,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:44:30,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:31,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:44:31,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:44:31,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:32,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:32,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:33,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:34,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:35,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:36,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:37,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:38,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:38,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:39,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:40,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:41,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:42,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:43,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:43,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:44,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:45,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:46,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:44:46,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:44:46,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:47,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:48,052 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:44:48,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:49,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:49,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:50,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:51,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:52,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:53,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:54,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:44:54,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:55,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:56,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:57,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:58,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:44:58,527 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:44:58,527 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:44:58,528 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:44:58,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:44:59,524 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:44:59,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:00,525 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:45:00,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:00,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:01,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:45:01,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:45:01,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:02,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:03,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:04,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:05,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:05,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:06,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:07,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:08,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:09,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:10,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:10,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:11,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:12,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:13,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:14,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:15,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:15,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:16,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:45:16,300 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:45:16,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:17,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:18,056 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:45:18,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:19,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:20,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:21,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:21,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:22,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:23,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:24,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:25,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:26,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:26,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:27,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:28,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:29,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:30,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:31,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:45:31,304 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:45:31,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:45:31,950 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:45:31,951 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:45:31,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:31,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:31,956 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:45:32,563 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:45:32,564 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:45:32,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:33,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:34,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:35,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:36,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:37,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:37,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:38,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:39,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:40,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:41,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:42,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:43,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:43,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:44,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:45,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:46,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:45:46,304 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:45:46,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:47,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:48,060 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:45:48,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:49,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:49,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:50,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:51,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:52,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:53,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:54,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:54,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:55,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:56,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:57,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:58,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:45:59,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:45:59,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:00,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:01,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:46:01,307 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:46:01,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:02,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:03,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:46:03,561 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:46:03,561 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:46:03,566 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:46:03,596 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:46:03,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:04,596 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:46:04,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:05,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:05,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:06,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:07,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:08,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:09,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:10,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:10,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:11,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:12,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:13,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:14,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:15,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:15,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:16,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:46:16,308 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:46:16,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:17,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:18,064 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:46:18,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:19,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:20,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:21,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:21,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:22,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:23,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:24,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:25,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:26,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:26,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:27,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:28,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:29,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:30,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:31,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:46:31,307 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:46:31,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:32,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:32,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:33,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:34,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:35,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:36,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:46:36,081 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:46:36,082 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:46:36,085 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:46:36,628 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:46:36,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:37,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:37,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:38,630 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:46:38,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:39,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:40,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:41,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:42,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:43,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:43,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:44,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:45,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:46,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:46:46,308 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:46:46,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:47,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:48,069 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:46:48,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:49,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:49,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:50,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:51,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:52,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:53,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:54,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:54,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:55,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:56,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:57,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:58,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:46:59,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:46:59,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:00,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:01,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:47:01,316 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:47:01,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:02,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:03,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:04,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:04,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:05,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:47:05,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:47:05,170 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:47:05,172 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:47:05,661 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:47:05,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:06,662 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:47:06,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:07,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:08,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:09,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:10,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:10,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:11,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:12,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:13,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:14,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:15,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:16,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:47:16,311 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:47:16,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:17,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:18,072 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:47:18,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:19,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:20,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:21,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:22,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:22,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:23,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:24,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:25,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:26,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:27,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:27,712 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:28,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:29,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:30,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:31,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:31,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:47:31,319 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:47:32,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:32,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:33,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:34,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:34,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:47:34,727 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:47:34,727 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:47:34,728 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:47:35,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:35,712 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:47:36,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:36,712 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:47:37,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:38,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:38,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:39,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:40,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:41,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:42,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:43,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:44,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:44,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:45,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:46,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:46,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:47:46,397 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:47:47,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:48,076 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:47:48,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:49,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:50,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:50,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:51,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:52,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:53,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:54,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:55,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:55,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:47:56,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:57,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:58,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:47:59,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:00,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:00,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:01,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:01,396 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:48:01,396 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:48:02,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:03,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:04,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:48:04,131 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:48:04,131 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:48:04,133 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:48:04,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:04,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:48:04,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:48:05,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:06,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:06,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:07,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:08,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:09,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:10,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:11,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:11,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:12,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:13,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:14,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:15,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:16,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:48:16,527 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:48:16,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:17,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:17,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:18,081 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:48:18,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:19,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:20,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:21,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:22,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:22,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:23,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:24,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:25,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:26,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:27,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:28,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:28,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:29,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:30,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:31,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:48:31,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:48:31,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:32,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:33,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:33,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:34,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:48:34,310 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:48:34,311 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:48:34,314 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:48:34,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:34,780 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:48:35,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:36,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:36,781 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:48:37,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:38,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:39,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:39,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:40,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:41,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:42,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:43,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:44,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:45,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:45,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:46,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:48:46,448 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:48:46,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:47,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:48,085 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:48:48,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:49,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:50,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:51,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:51,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:52,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:53,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:54,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:55,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:56,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:48:56,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:57,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:58,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:48:59,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:00,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:01,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:49:01,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:49:01,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:02,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:02,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:03,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:04,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:05,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:06,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:49:06,627 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:49:06,627 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:49:06,628 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:49:06,812 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:49:07,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:07,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:07,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:08,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:08,814 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:49:09,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:10,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:11,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:12,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:12,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:13,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:14,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:15,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:16,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:49:16,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:49:16,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:17,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:17,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:18,090 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:49:18,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:19,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:20,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:21,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:22,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:22,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:23,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:24,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:25,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:26,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:27,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:28,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:28,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:29,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:30,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:31,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:49:31,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:49:31,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:32,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:33,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:33,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:34,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:35,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:36,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:37,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:38,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:49:38,784 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:49:38,785 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:49:38,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:38,787 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:49:38,852 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:49:39,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:40,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:40,854 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:49:41,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:42,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:43,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:44,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:44,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:45,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:46,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:46,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:49:46,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:49:47,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:48,093 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:49:48,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:49,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:50,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:50,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:51,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:52,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:53,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:54,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:55,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:55,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:49:56,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:57,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:58,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:49:59,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:00,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:01,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:01,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:50:01,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:50:01,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:02,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:03,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:04,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:05,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:06,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:06,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:07,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:07,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:50:07,726 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:50:07,726 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:50:07,727 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:50:07,891 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:50:08,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:08,892 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:50:09,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:10,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:11,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:12,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:12,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:13,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:14,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:15,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:16,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:16,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:50:16,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:50:17,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:17,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:18,097 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:50:18,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:19,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:20,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:21,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:22,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:22,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:23,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:24,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:25,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:26,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:27,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:28,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:28,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:29,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:30,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:31,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:31,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:50:31,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:50:32,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:33,368 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:33,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:34,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:35,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:36,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:37,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:38,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:38,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:50:38,451 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:50:38,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:50:38,454 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:50:38,923 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:50:39,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:39,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:40,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:40,924 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:50:41,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:42,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:43,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:44,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:44,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:45,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:46,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:46,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:50:46,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:50:47,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:48,101 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:50:48,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:49,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:50,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:50,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:51,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:52,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:53,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:54,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:55,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:55,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:50:56,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:57,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:58,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:50:59,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:00,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:01,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:01,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:51:01,451 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:51:01,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:02,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:03,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:04,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:05,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:06,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:06,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:07,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:08,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:09,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:10,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:11,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:11,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:12,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:12,572 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:51:12,574 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:51:12,574 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:51:12,576 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:51:12,962 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:51:13,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:14,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:14,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:51:15,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:16,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:16,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:51:16,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:51:17,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:17,602 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:18,105 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:51:18,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:19,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:20,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:21,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:22,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:22,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:23,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:24,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:25,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:26,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:27,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:27,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:28,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:29,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:30,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:31,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:31,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:51:31,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:51:32,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:33,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:33,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:34,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:35,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:36,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:37,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:38,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:38,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:39,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:40,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:41,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:42,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:43,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:51:43,069 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:51:43,069 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:51:43,071 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:51:43,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:43,992 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:51:44,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:44,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:44,994 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:51:45,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:46,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:51:46,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:51:46,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:47,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:48,109 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:51:48,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:49,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:49,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:50,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:51,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:52,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:53,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:54,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:54,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:51:55,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:56,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:57,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:58,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:59,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:51:59,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:00,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:01,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:52:01,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:52:01,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:02,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:03,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:04,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:05,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:05,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:06,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:07,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:08,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:09,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:10,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:11,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:11,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:12,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:13,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:52:13,508 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:52:13,508 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:52:13,510 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:52:13,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:14,030 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:52:14,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:15,031 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:52:15,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:16,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:52:16,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:52:16,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:16,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:17,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:18,114 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:52:18,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:19,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:20,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:21,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:22,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:22,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:23,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:24,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:25,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:26,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:27,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:28,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:28,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:29,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:30,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:31,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:52:31,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:52:31,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:32,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:33,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:34,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:34,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:35,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:36,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:37,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:38,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:39,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:39,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:40,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:41,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:52:41,542 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:52:41,542 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:52:41,544 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:52:41,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:42,064 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:52:42,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:43,065 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:52:43,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:44,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:45,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:45,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:46,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:52:46,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:52:46,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:47,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:48,117 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:52:48,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:49,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:50,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:50,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:51,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:52,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:53,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:54,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:55,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:56,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:52:56,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:57,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:58,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:52:59,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:00,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:01,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:01,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:53:01,452 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:53:01,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:02,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:03,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:04,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:05,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:06,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:07,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:07,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:08,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:09,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:10,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:11,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:53:11,195 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:53:11,196 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:53:11,198 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:53:11,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:12,095 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:53:12,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:12,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:13,096 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:53:13,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:14,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:15,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:16,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:53:16,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:53:16,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:17,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:17,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:18,120 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:53:18,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:19,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:20,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:21,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:22,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:23,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:23,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:24,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:25,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:26,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:27,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:28,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:29,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:29,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:30,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:31,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:53:31,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:53:31,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:32,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:33,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:34,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:35,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:35,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:36,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:37,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:38,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:39,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:40,023 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:40,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:41,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:42,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:42,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:53:42,999 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:53:43,000 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:53:43,001 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:53:43,129 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:53:43,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:44,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:45,131 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:53:45,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:45,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:46,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:53:46,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:53:46,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:47,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:48,124 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:53:48,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:49,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:50,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:51,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:51,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:52,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:53,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:54,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:55,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:56,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:57,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:53:57,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:58,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:53:59,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:00,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:01,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:54:01,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:54:01,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:02,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:02,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:03,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:04,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:05,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:06,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:07,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:08,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:08,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:09,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:10,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:11,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:12,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:13,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:13,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:54:13,611 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:54:13,612 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:54:13,613 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:54:14,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:14,163 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:54:15,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:15,164 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:54:16,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:16,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:54:16,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:54:17,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:18,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:18,128 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:54:19,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:19,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:20,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:21,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:22,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:23,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:24,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:25,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:25,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:26,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:27,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:28,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:29,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:30,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:30,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:31,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:31,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:54:31,453 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:54:32,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:33,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:34,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:35,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:36,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:36,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:37,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:38,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:39,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:40,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:41,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:41,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:42,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:43,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:44,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:45,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:46,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:46,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:46,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:54:46,443 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:54:46,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:54:46,446 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:54:46,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:54:46,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:54:47,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:47,194 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:54:47,195 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:54:48,133 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:54:48,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:49,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:50,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:51,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:52,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:52,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:53,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:54,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:55,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:56,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:57,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:57,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:54:58,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:54:59,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:00,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:01,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:01,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:55:01,590 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:55:02,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:02,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:03,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:04,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:05,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:06,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:07,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:08,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:08,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:09,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:10,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:11,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:12,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:13,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:13,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:14,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:15,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:16,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:16,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:55:16,596 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:55:17,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:18,137 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:55:18,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:18,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:55:18,228 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:55:18,228 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:55:18,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:18,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:55:19,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:19,226 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:55:19,227 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:55:20,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:21,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:22,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:23,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:24,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:24,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:25,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:26,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:27,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:28,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:29,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:30,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:30,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:31,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:31,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:55:31,594 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:55:32,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:33,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:34,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:35,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:35,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:36,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:37,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:38,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:39,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:40,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:41,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:41,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:42,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:43,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:44,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:45,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:46,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:46,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:46,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:55:46,600 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:55:47,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:48,138 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:55:48,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:48,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:55:48,784 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:55:48,784 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:55:48,786 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:55:49,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:49,252 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:55:49,252 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:55:50,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:51,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:51,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:52,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:53,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:54,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:55,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:56,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:57,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:55:57,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:58,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:55:59,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:00,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:01,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:01,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:56:01,676 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:56:02,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:02,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:03,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:04,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:05,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:06,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:07,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:08,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:08,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:09,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:10,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:11,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:12,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:13,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:13,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:14,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:15,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:16,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:16,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:56:16,676 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:56:17,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:18,142 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:56:18,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:18,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:18,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:56:18,714 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:56:18,714 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:56:18,715 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:56:19,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:19,281 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:56:19,281 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:56:20,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:21,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:22,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:23,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:23,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:24,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:25,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:26,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:27,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:28,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:28,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:29,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:30,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:31,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:31,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:56:31,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:56:32,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:33,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:34,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:34,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:35,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:36,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:37,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:38,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:39,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:39,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:40,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:41,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:42,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:43,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:44,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:45,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:45,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:46,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:46,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:56:46,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:56:47,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:48,153 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:56:48,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:49,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:50,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:50,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:51,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:52,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:52,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:56:52,853 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:56:52,853 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:56:52,854 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:56:53,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:53,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:56:53,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:56:54,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:55,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:55,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:56:56,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:57,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:58,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:56:59,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:00,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:01,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:01,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:01,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:57:01,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:57:02,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:03,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:04,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:05,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:06,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:07,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:07,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:08,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:09,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:10,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:11,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:12,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:12,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:13,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:14,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:15,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:16,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:16,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:57:16,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:57:17,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:17,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:18,151 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:57:18,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:19,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:20,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:21,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:22,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:23,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:23,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:24,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:25,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:57:25,071 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:57:25,072 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:57:25,077 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:57:25,343 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:57:25,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:26,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:27,345 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:57:27,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:28,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:29,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:29,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:30,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:31,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:31,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:57:31,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:57:32,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:33,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:34,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:35,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:35,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:36,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:37,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:38,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:39,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:40,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:40,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:41,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:42,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:43,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:44,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:45,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:46,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:46,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:46,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:57:46,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:57:47,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:48,158 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:57:48,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:49,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:50,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:51,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:51,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:52,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:57:52,154 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:57:52,154 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:57:52,156 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:57:52,372 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:57:52,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:53,373 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:57:53,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:54,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:55,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:56,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:57:56,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:57,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:58,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:57:59,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:00,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:01,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:01,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:58:01,679 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:58:01,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:02,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:03,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:04,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:05,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:06,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:07,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:07,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:08,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:09,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:10,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:11,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:12,274 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:12,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:13,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:14,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:15,575 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:16,576 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:16,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:58:16,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:58:17,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:17,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:18,159 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:58:18,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:19,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:20,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:21,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:22,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:22,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:58:22,943 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:58:22,944 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:58:22,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:22,945 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:58:23,405 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:58:23,405 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:58:23,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:24,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:25,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:26,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:27,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:28,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:28,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:29,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:30,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:31,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:31,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:58:31,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:58:32,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:33,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:34,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:34,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:35,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:36,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:37,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:38,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:39,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:39,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:40,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:41,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:42,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:43,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:44,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:45,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:45,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:46,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:46,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:58:46,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:58:47,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:48,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:58:48,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:49,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:50,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:51,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:51,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:52,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:52,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:58:52,803 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:58:52,804 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:58:52,805 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:58:53,435 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:58:53,436 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:58:53,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:54,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:55,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:56,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:58:56,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:57,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:58,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:58:59,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:00,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:01,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:01,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:59:01,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:59:01,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:02,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:03,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:04,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:05,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:06,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:07,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:07,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:08,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:09,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:10,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:11,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:12,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:12,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:13,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:14,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:15,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:16,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:16,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:59:16,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:59:17,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:17,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:18,172 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:59:18,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:19,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:20,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:21,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:22,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:23,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:23,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:24,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:59:24,103 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:59:24,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:59:24,104 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:59:24,465 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:59:24,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:25,466 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:59:25,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:26,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:27,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:28,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:29,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:29,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:30,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:31,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:31,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:59:31,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:59:32,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:33,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:34,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:35,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:35,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:36,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:37,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:38,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:39,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:40,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:40,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:41,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:42,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:43,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:44,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:45,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:46,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:46,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:46,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 17:59:46,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 17:59:47,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:48,169 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 17:59:48,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:49,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:50,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:51,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:51,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:52,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:53,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:54,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:55,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 17:59:55,137 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 17:59:55,137 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 17:59:55,139 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 17:59:55,497 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 17:59:55,497 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 17:59:55,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:56,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:57,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 17:59:57,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:58,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 17:59:59,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:00,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:01,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:01,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:00:01,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:00:02,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:02,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:03,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:04,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:05,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:06,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:07,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:08,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:08,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:09,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:10,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:11,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:12,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:13,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:13,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:14,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:15,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:16,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:00:16,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:00:16,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:17,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:18,174 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:00:18,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:19,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:19,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:20,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:21,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:22,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:23,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:24,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:24,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:25,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:00:25,049 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:00:25,049 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:00:25,050 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:00:25,524 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:00:25,525 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:00:25,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:26,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:27,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:28,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:29,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:30,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:30,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:31,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:00:31,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:00:31,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:32,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:33,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:34,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:35,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:36,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:36,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:37,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:38,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:39,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:40,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:41,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:41,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:42,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:43,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:44,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:45,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:46,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:46,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:00:46,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:00:46,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:47,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:48,178 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:00:48,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:49,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:50,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:51,730 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:52,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:52,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:53,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:54,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:55,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:56,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:57,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:57,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:00:57,904 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:00:57,905 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:00:57,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:00:57,909 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:00:58,553 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:00:58,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:00:59,554 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:00:59,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:00,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:01,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:01:01,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:01:01,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:02,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:03,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:03,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:04,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:05,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:06,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:07,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:08,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:08,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:09,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:10,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:11,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:12,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:13,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:13,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:14,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:15,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:16,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:01:16,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:01:16,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:17,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:18,183 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:01:18,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:19,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:19,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:20,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:21,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:22,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:23,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:24,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:24,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:25,736 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:26,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:01:26,615 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:01:26,615 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:01:26,617 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:01:27,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:27,585 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:01:27,585 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:01:28,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:29,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:30,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:30,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:31,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:31,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:01:31,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:01:32,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:33,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:34,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:35,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:36,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:36,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:37,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:38,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:39,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:40,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:41,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:41,528 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:42,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:43,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:44,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:45,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:46,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:46,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:01:46,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:01:46,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:47,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:48,187 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:01:48,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:49,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:50,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:51,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:52,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:52,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:53,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:54,536 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:55,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:56,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:56,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:01:56,952 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:01:56,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:01:56,955 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:01:57,613 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:01:57,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:01:58,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:01:58,614 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:01:59,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:00,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:01,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:01,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:02:01,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:02:02,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:03,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:03,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:04,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:05,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:06,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:07,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:08,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:08,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:09,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:10,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:11,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:12,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:13,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:13,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:14,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:15,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:16,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:16,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:02:16,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:02:17,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:18,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:18,191 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:02:19,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:19,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:20,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:21,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:22,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:23,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:24,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:24,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:25,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:26,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:27,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:28,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:29,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:29,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:29,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:02:29,657 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:02:29,658 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:02:29,660 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:02:30,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:30,648 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:02:31,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:31,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:02:31,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:02:32,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:32,650 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:02:33,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:34,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:35,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:35,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:36,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:37,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:38,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:39,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:40,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:40,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:41,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:42,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:43,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:44,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:45,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:45,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:46,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:46,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:02:46,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:02:47,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:48,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:48,196 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:02:49,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:50,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:51,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:51,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:52,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:53,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:54,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:55,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:56,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:56,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:02:57,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:58,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:59,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:02:59,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:02:59,209 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:02:59,209 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:02:59,210 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:02:59,678 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:03:00,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:00,679 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:03:01,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:01,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:03:01,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:03:01,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:02,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:03,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:04,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:05,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:06,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:07,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:07,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:08,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:09,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:10,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:11,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:12,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:13,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:13,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:14,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:15,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:16,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:16,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:03:16,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:03:17,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:18,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:18,199 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:03:19,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:19,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:20,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:21,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:22,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:23,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:24,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:24,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:25,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:26,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:27,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:28,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:29,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:30,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:03:30,048 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:03:30,049 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:03:30,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:30,050 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:03:30,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:30,707 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:03:31,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:31,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:03:31,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:03:32,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:32,709 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:03:33,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:34,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:35,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:35,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:36,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:37,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:38,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:39,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:40,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:41,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:41,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:42,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:43,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:44,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:45,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:46,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:46,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:46,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:03:46,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:03:47,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:48,203 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:03:48,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:49,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:50,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:51,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:52,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:52,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:53,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:54,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:55,461 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:56,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:57,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:03:57,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:58,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:03:59,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:00,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:00,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:04:00,835 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:04:00,835 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:04:00,836 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:04:01,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:01,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:04:01,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:04:01,738 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:04:02,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:02,739 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:04:02,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:03,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:04,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:05,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:06,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:07,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:08,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:08,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:09,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:10,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:11,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:12,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:13,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:13,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:14,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:15,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:16,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:16,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:04:16,682 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:04:17,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:18,207 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:04:18,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:19,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:19,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:20,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:21,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:22,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:23,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:24,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:25,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:25,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:26,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:27,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:28,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:29,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:30,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:30,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:30,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:04:30,758 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:04:30,758 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:04:30,759 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:04:30,769 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:04:31,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:31,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:04:31,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:04:32,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:32,771 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:04:33,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:34,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:35,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:36,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:36,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:37,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:38,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:39,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:40,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:41,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:42,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:42,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:43,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:44,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:45,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:46,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:46,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:04:46,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:04:47,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:47,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:48,211 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:04:48,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:49,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:50,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:51,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:52,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:53,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:53,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:54,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:55,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:56,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:57,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:58,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:04:58,489 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:04:59,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:00,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:01,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:05:01,044 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:05:01,044 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:05:01,046 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:05:01,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:01,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:05:01,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:05:01,803 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:05:02,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:02,805 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:05:03,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:03,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:04,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:05,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:06,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:07,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:08,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:08,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:09,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:10,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:11,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:12,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:13,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:14,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:14,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:15,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:16,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:16,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:05:16,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:05:17,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:18,214 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:05:18,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:19,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:19,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:20,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:21,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:22,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:23,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:24,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:24,634 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:25,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:26,644 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:27,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:28,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:29,435 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:29,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:30,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:05:30,596 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:05:30,596 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:05:30,597 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:05:30,832 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:05:31,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:31,682 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:05:31,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:05:32,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:32,833 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:05:33,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:34,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:35,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:35,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:36,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:37,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:38,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:39,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:40,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:41,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:41,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:42,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:43,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:44,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:45,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:46,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:46,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:46,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:05:46,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:05:47,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:48,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:48,219 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:05:49,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:50,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:51,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:51,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:52,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:53,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:54,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:55,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:56,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:57,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:57,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:05:58,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:05:59,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:00,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:01,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:01,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:06:01,689 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:06:02,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:02,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:02,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:06:02,882 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:06:02,882 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:06:02,883 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:06:03,333 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:03,865 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:06:04,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:04,865 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:06:05,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:06,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:07,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:08,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:08,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:09,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:10,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:11,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:12,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:13,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:14,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:14,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:15,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:16,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:16,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:06:16,689 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:06:17,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:18,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:18,222 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:06:19,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:19,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:20,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:21,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:22,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:23,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:24,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:24,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:25,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:26,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:27,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:28,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:29,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:29,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:30,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:31,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:31,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:06:31,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:06:32,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:33,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:34,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:34,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:35,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:35,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:06:35,689 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:06:35,689 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:06:35,691 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:06:35,899 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:06:36,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:36,900 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:06:37,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:38,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:39,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:40,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:40,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:41,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:42,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:43,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:44,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:45,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:46,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:46,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:46,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:06:46,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:06:47,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:48,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:48,226 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:06:49,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:50,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:51,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:51,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:52,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:53,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:54,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:55,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:56,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:56,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:06:57,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:58,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:06:59,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:00,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:01,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:01,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:01,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:07:01,695 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:07:02,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:03,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:04,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:05,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:05,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:07:05,962 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:07:05,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:07:05,964 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:07:06,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:06,922 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:07:07,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:07,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:08,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:08,923 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:07:09,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:10,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:11,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:12,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:12,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:13,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:14,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:15,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:16,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:16,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:07:16,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:07:17,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:17,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:18,229 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:07:18,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:19,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:20,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:21,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:22,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:23,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:23,468 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:24,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:25,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:26,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:27,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:28,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:29,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:29,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:30,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:31,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:31,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:07:31,695 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:07:32,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:33,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:34,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:34,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:35,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:36,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:36,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:07:36,628 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:07:36,628 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:07:36,630 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:07:36,952 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:07:37,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:38,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:38,954 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:07:39,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:40,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:40,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:41,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:42,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:43,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:44,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:45,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:46,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:46,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:46,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:07:46,692 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:07:47,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:48,233 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:07:48,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:49,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:50,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:51,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:51,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:52,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:53,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:54,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:55,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:56,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:57,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:07:57,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:58,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:07:59,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:00,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:01,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:01,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:08:01,695 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:08:02,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:02,862 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:03,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:03,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:08:03,846 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:08:03,847 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:08:03,848 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:08:03,977 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:08:04,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:04,978 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:08:05,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:06,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:07,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:07,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:08,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:09,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:10,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:11,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:12,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:13,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:13,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:14,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:15,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:16,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:16,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:08:16,693 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:08:17,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:18,237 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:08:18,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:18,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:19,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:20,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:21,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:22,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:23,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:24,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:24,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:25,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:26,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:27,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:28,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:29,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:29,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:30,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:31,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:31,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:08:31,698 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:08:32,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:33,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:34,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:34,533 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:35,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:36,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:37,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:37,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:08:37,641 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:08:37,642 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:08:37,644 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:08:38,011 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:08:38,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:39,012 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:08:39,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:40,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:40,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:41,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:42,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:43,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:44,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:45,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:46,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:46,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:46,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:08:46,695 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:08:47,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:48,240 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:08:48,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:49,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:50,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:51,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:52,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:52,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:53,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:54,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:55,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:56,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:57,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:08:57,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:58,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:08:59,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:00,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:01,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:01,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:09:01,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:09:02,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:02,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:03,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:04,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:05,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:06,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:07,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:08,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:08,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:09:08,657 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:09:08,657 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:09:08,658 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:09:09,043 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:09:09,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:09,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:10,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:11,045 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:09:11,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:12,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:13,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:13,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:14,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:15,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:16,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:09:16,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:09:16,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:17,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:18,244 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:09:18,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:19,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:19,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:20,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:21,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:22,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:23,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:24,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:24,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:25,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:26,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:27,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:28,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:29,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:29,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:30,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:31,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:09:31,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:09:31,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:32,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:33,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:34,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:34,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:35,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:36,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:37,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:38,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:39,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:39,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:40,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:09:40,151 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:09:40,152 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:09:40,154 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:09:40,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:41,083 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:09:41,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:42,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:43,084 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:09:43,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:44,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:45,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:45,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:46,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:09:46,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:09:46,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:47,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:48,248 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:09:48,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:49,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:50,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:51,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:51,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:52,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:53,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:54,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:55,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:56,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:57,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:09:57,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:58,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:09:59,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:00,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:01,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:10:01,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:10:01,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:02,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:02,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:03,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:04,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:05,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:06,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:07,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:08,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:08,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:09,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:10,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:11,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:12,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:13,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:14,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:10:14,014 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:10:14,014 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:10:14,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:14,016 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:10:14,120 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:10:14,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:15,122 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:10:15,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:16,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:10:16,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:10:16,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:17,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:18,252 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:10:18,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:19,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:19,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:20,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:21,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:22,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:23,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:24,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:25,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:25,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:26,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:27,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:28,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:29,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:30,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:30,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:31,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:10:31,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:10:31,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:32,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:33,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:34,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:35,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:36,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:36,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:37,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:38,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:39,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:40,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:41,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:41,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:42,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:43,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:44,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:45,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:46,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:10:46,548 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:10:46,548 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:10:46,550 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:10:46,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:46,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:10:46,969 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:10:47,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:47,153 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:10:47,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:48,256 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:10:48,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:49,154 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:10:49,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:50,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:51,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:52,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:52,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:53,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:54,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:55,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:56,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:57,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:58,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:10:58,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:10:59,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:01,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:01,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:11:01,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:11:02,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:02,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:03,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:03,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:04,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:05,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:06,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:07,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:08,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:08,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:09,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:10,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:11,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:12,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:13,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:13,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:14,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:15,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:16,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:11:16,968 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:11:17,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:17,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:18,261 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:11:18,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:18,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:11:18,871 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:11:18,871 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:11:18,873 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:11:19,184 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:11:19,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:20,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:21,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:21,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:11:22,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:23,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:24,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:24,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:25,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:26,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:27,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:28,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:29,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:29,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:30,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:31,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:31,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:11:31,965 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:11:32,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:33,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:34,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:35,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:35,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:36,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:37,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:38,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:39,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:40,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:41,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:41,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:42,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:43,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:44,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:45,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:46,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:46,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:46,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:11:46,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:11:47,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:48,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:48,261 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:11:49,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:49,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:11:49,410 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:11:49,410 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:11:49,415 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:11:50,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:50,215 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:11:51,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:51,217 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:11:52,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:52,451 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:53,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:54,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:55,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:56,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:57,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:57,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:11:58,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:11:59,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:00,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:01,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:01,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:12:01,968 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:12:02,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:03,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:03,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:04,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:05,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:06,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:07,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:08,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:08,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:09,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:10,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:11,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:12,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:13,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:14,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:14,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:15,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:16,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:16,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:12:16,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:12:17,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:18,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:18,272 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:12:19,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:20,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:20,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:20,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:12:20,767 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:12:20,767 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:12:20,770 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:12:21,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:21,247 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:12:22,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:23,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:23,249 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:12:24,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:25,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:25,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:26,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:27,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:28,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:29,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:30,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:31,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:31,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:31,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:12:31,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:12:32,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:33,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:34,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:35,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:36,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:36,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:37,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:38,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:39,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:40,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:41,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:41,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:42,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:43,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:44,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:45,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:46,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:46,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:46,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:12:46,978 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:12:47,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:48,198 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:48,278 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:12:49,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:50,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:51,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:51,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:51,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:12:51,655 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:12:51,655 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:12:51,660 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:12:52,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:52,280 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:12:53,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:53,281 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:12:54,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:55,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:56,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:57,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:57,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:12:58,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:12:59,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:00,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:01,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:01,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:13:01,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:13:02,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:03,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:03,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:04,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:05,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:06,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:07,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:08,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:08,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:09,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:10,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:11,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:12,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:13,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:14,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:14,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:15,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:16,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:16,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:13:16,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:13:17,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:18,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:18,280 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:13:19,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:19,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:20,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:21,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:22,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:23,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:23,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:13:23,429 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:13:23,430 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:13:23,434 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:13:24,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:24,308 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:13:24,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:25,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:25,309 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:13:26,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:27,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:28,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:29,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:30,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:30,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:31,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:31,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:13:31,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:13:32,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:33,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:34,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:35,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:36,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:36,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:37,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:38,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:39,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:40,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:41,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:41,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:42,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:43,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:44,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:45,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:46,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:46,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:13:46,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:13:47,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:47,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:48,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:48,280 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:13:49,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:50,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:51,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:52,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:52,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:53,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:54,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:55,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:13:55,064 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:13:55,064 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:13:55,066 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:13:55,340 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:13:55,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:56,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:57,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:57,342 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:13:58,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:13:58,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:13:59,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:00,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:01,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:01,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:14:01,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:14:02,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:03,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:03,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:04,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:05,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:06,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:07,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:08,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:08,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:09,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:10,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:11,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:12,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:13,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:14,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:14,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:15,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:16,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:16,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:14:16,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:14:17,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:18,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:18,283 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:14:19,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:19,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:20,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:21,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:22,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:23,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:24,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:24,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:14:24,577 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:14:24,578 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:14:24,580 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:14:25,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:25,368 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:14:25,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:26,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:27,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:27,370 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:14:28,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:29,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:30,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:30,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:31,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:31,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:14:31,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:14:32,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:33,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:34,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:35,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:35,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:36,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:37,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:38,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:39,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:40,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:40,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:41,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:42,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:43,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:44,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:45,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:46,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:46,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:46,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:14:46,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:14:47,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:48,288 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:14:48,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:49,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:50,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:51,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:51,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:52,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:53,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:54,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:54,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:14:54,851 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:14:54,852 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:14:54,853 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:14:55,400 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:14:55,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:56,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:56,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:14:57,403 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:14:57,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:58,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:14:59,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:00,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:01,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:01,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:15:01,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:15:02,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:02,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:03,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:04,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:05,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:06,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:07,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:07,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:08,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:09,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:10,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:11,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:12,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:12,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:13,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:14,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:15,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:16,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:16,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:15:16,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:15:17,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:18,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:18,292 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:15:18,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:19,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:20,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:21,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:22,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:23,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:23,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:24,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:25,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:26,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:27,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:28,326 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:15:28,326 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:15:28,327 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:15:28,331 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:15:28,447 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:15:28,741 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:28,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:29,448 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:15:29,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:30,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:31,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:31,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:15:31,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:15:32,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:33,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:34,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:34,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:35,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:36,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:37,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:38,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:39,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:40,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:40,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:41,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:42,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:43,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:44,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:45,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:45,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:46,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:46,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:15:46,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:15:47,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:48,296 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:15:48,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:49,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:50,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:51,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:51,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:52,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:53,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:54,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:55,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:56,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:56,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:15:57,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:57,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:15:57,649 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:15:57,649 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:15:57,651 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:15:58,480 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:15:58,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:59,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:15:59,481 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:16:00,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:01,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:01,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:16:01,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:16:02,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:02,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:03,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:04,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:05,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:06,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:07,480 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:07,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:08,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:09,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:10,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:11,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:12,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:13,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:13,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:14,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:15,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:16,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:16,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:16:16,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:16:17,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:18,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:16:18,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:19,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:19,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:20,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:21,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:22,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:23,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:24,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:25,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:25,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:26,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:27,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:28,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:29,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:30,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:30,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:31,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:16:31,309 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:16:31,309 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:16:31,314 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:16:31,509 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:16:31,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:31,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:16:31,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:16:32,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:33,510 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:16:33,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:34,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:35,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:36,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:36,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:37,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:38,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:39,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:40,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:41,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:42,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:42,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:43,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:44,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:45,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:46,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:46,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:16:46,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:16:47,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:48,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:48,304 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:16:48,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:49,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:50,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:51,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:52,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:53,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:53,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:54,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:55,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:56,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:57,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:58,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:16:58,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:16:59,735 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:00,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:01,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:01,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:17:01,982 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:17:02,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:03,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:17:03,555 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:17:03,555 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:17:03,559 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:17:03,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:03,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:04,545 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:17:04,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:05,546 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:17:05,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:06,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:07,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:08,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:09,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:09,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:10,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:11,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:12,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:13,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:14,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:15,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:15,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:16,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:16,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:17:16,981 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:17:17,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:18,307 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:17:18,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:19,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:20,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:21,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:21,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:22,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:23,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:24,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:25,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:26,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:27,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:27,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:28,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:29,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:30,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:31,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:31,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:17:31,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:17:32,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:33,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:33,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:34,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:35,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:36,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:17:36,083 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:17:36,083 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:17:36,089 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:17:36,576 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:17:36,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:37,577 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:17:37,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:38,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:38,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:39,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:40,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:41,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:42,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:43,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:43,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:44,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:45,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:46,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:46,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:17:46,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:17:47,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:48,311 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:17:48,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:49,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:49,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:50,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:51,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:52,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:53,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:54,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:17:54,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:55,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:56,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:57,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:58,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:17:59,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:00,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:00,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:01,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:01,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:18:01,996 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:18:02,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:03,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:04,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:05,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:05,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:06,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:07,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:18:07,433 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:18:07,433 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:18:07,438 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:18:07,611 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:18:07,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:08,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:09,613 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:18:09,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:10,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:11,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:11,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:12,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:13,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:14,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:15,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:16,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:18:16,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:18:17,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:17,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:17,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:18,315 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:18:18,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:19,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:20,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:21,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:22,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:22,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:23,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:24,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:25,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:26,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:27,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:27,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:28,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:29,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:30,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:31,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:18:31,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:18:32,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:32,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:33,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:33,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:34,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:35,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:37,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:38,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:38,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:38,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:39,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:18:39,150 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:18:39,150 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:18:39,152 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:18:39,644 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:18:40,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:41,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:41,646 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:18:42,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:43,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:44,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:44,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:45,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:46,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:46,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:18:46,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:18:47,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:48,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:48,319 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:18:49,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:50,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:50,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:51,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:52,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:53,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:54,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:55,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:55,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:18:56,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:57,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:58,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:18:59,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:00,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:01,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:01,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:19:01,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:19:02,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:02,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:03,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:04,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:05,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:06,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:07,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:08,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:08,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:09,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:09,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:19:09,920 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:19:09,920 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:19:09,922 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:19:10,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:10,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:19:11,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:11,681 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:19:12,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:13,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:13,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:14,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:15,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:16,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:16,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:19:16,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:19:17,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:18,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:18,324 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:19:18,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:19,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:20,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:21,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:22,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:23,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:23,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:24,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:25,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:26,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:27,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:28,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:29,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:29,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:30,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:31,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:31,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:19:31,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:19:32,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:33,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:34,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:35,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:35,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:36,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:37,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:38,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:39,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:40,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:40,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:19:40,872 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:19:40,873 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:19:40,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:40,876 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:19:41,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:41,712 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:19:41,712 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:19:42,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:43,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:44,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:45,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:45,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:46,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:46,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:19:46,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:19:47,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:48,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:48,328 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:19:49,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:50,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:51,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:51,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:52,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:53,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:54,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:55,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:56,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:56,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:19:57,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:58,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:19:59,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:00,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:01,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:01,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:20:01,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:20:02,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:02,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:03,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:04,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:05,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:06,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:07,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:08,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:08,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:09,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:10,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:11,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:12,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:12,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:20:12,619 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:20:12,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:20:12,624 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:20:12,744 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:20:13,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:13,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:13,746 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:20:14,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:15,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:16,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:16,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:20:16,988 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:20:17,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:18,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:18,332 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:20:19,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:19,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:20,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:21,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:22,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:23,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:24,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:24,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:25,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:26,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:27,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:28,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:29,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:30,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:30,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:31,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:31,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:20:31,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:20:32,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:33,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:34,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:35,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:35,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:36,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:37,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:38,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:39,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:40,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:40,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:20:40,582 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:20:40,582 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:20:40,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:40,584 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:20:40,775 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:20:41,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:41,776 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:20:42,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:43,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:44,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:45,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:46,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:46,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:46,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:20:46,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:20:47,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:48,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:48,336 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:20:49,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:50,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:51,310 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:52,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:52,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:53,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:54,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:55,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:56,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:57,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:20:57,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:58,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:20:59,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:00,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:01,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:01,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:21:01,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:21:02,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:02,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:03,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:04,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:05,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:06,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:07,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:08,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:08,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:09,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:10,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:11,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:21:11,290 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:21:11,290 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:21:11,292 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:21:11,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:11,812 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:21:11,813 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:21:12,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:13,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:13,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:14,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:15,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:16,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:16,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:21:16,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:21:17,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:18,340 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:21:18,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:18,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:19,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:20,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:21,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:22,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:23,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:24,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:24,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:25,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:26,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:27,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:28,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:29,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:29,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:30,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:31,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:31,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:21:31,989 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:21:32,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:33,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:34,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:35,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:35,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:36,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:37,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:38,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:39,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:40,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:40,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:41,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:42,582 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:43,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:44,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:45,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:45,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:45,591 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:21:45,592 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:21:45,592 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:21:45,593 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:21:45,852 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:21:46,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:46,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:21:46,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:21:47,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:47,854 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:21:48,345 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:21:48,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:49,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:50,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:50,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:51,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:52,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:53,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:54,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:55,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:56,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:21:56,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:57,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:58,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:21:59,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:00,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:01,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:01,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:01,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:22:01,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:22:02,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:03,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:04,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:05,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:06,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:07,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:07,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:08,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:09,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:10,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:11,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:12,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:12,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:13,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:14,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:22:14,364 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:22:14,365 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:22:14,366 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:22:14,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:14,880 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:22:15,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:15,881 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:22:16,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:16,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:22:16,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:22:17,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:17,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:18,348 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:22:18,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:19,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:20,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:21,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:22,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:23,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:23,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:24,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:25,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:26,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:27,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:28,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:29,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:29,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:30,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:31,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:31,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:22:31,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:22:32,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:33,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:34,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:35,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:35,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:36,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:37,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:38,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:39,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:40,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:40,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:41,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:42,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:43,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:43,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:22:43,831 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:22:43,831 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:22:43,833 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:22:43,911 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:22:44,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:45,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:45,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:45,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:22:46,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:46,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:22:46,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:22:47,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:48,352 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:22:48,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:49,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:50,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:51,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:51,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:52,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:53,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:54,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:55,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:56,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:22:56,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:57,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:58,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:22:59,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:00,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:01,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:01,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:23:01,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:23:02,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:02,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:03,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:04,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:05,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:06,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:07,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:07,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:08,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:09,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:10,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:11,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:12,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:13,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:13,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:14,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:23:14,078 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:23:14,078 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:23:14,080 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:23:14,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:14,943 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:23:15,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:15,945 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:23:16,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:16,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:23:16,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:23:17,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:18,356 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:23:18,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:18,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:19,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:20,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:21,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:22,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:23,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:23,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:24,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:25,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:26,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:27,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:28,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:28,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:29,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:30,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:31,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:31,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:23:31,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:23:32,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:33,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:34,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:34,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:35,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:36,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:37,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:38,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:39,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:39,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:40,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:41,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:42,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:43,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:44,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:44,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:45,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:46,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:23:46,288 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:23:46,289 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:23:46,290 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:23:46,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:46,980 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:23:46,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:23:46,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:23:47,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:47,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:23:48,360 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:23:48,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:49,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:50,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:50,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:51,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:52,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:53,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:54,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:55,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:23:55,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:56,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:57,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:58,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:23:59,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:00,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:00,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:01,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:01,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:24:01,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:24:02,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:03,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:04,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:05,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:06,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:06,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:07,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:08,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:09,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:10,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:11,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:11,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:12,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:13,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:14,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:15,696 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:24:15,698 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:24:15,698 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:24:15,700 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:24:16,025 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:24:16,026 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:24:16,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:16,738 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:16,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:16,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:24:16,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:24:17,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:18,364 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:24:18,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:19,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:20,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:21,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:22,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:22,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:23,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:24,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:25,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:26,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:27,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:28,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:28,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:29,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:30,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:31,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:31,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:24:31,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:24:32,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:33,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:34,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:34,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:35,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:36,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:37,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:38,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:39,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:39,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:40,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:41,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:42,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:43,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:44,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:44,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:45,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:46,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:46,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:24:46,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:24:47,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:24:47,069 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:24:47,070 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:24:47,071 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:24:47,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:48,066 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:24:48,066 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:24:48,369 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:24:48,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:49,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:49,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:50,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:51,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:52,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:53,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:54,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:54,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:55,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:56,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:57,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:58,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:24:59,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:24:59,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:00,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:01,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:01,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:25:01,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:25:02,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:03,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:04,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:05,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:05,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:06,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:07,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:08,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:09,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:10,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:11,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:11,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:12,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:13,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:14,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:15,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:16,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:16,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:16,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:25:16,993 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:25:17,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:25:17,198 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:25:17,199 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:25:17,201 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:25:17,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:18,106 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:25:18,106 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:25:18,372 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:25:18,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:19,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:20,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:21,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:22,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:22,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:23,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:24,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:25,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:26,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:27,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:28,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:28,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:29,869 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:30,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:31,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:31,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:25:31,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:25:32,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:33,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:34,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:34,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:35,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:36,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:37,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:38,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:39,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:39,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:40,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:41,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:42,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:43,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:44,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:44,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:45,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:46,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:47,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:25:47,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:25:47,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:48,374 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:25:48,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:25:48,702 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:25:48,703 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:25:48,704 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:25:49,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:49,146 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:25:50,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:50,147 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:25:50,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:51,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:52,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:53,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:54,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:55,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:56,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:56,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:25:57,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:58,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:25:59,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:00,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:01,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:01,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:26:01,998 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:26:02,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:02,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:03,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:04,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:05,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:06,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:07,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:07,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:08,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:09,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:10,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:11,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:12,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:13,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:13,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:14,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:15,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:16,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:17,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:26:17,001 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:26:17,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:18,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:18,381 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:26:19,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:19,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:20,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:20,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:26:20,141 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:26:20,141 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:26:20,143 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:26:20,179 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:26:21,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:22,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:22,182 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:26:23,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:24,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:24,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:25,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:26,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:27,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:28,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:29,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:30,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:30,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:31,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:32,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:26:32,002 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:26:32,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:33,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:34,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:35,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:35,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:36,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:37,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:38,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:39,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:40,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:41,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:41,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:42,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:43,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:44,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:45,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:46,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:46,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:47,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:26:47,005 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:26:47,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:48,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:48,390 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:26:49,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:49,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:26:49,976 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:26:49,976 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:26:49,978 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:26:50,218 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:26:50,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:51,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:52,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:52,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:26:52,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:53,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:54,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:55,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:56,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:57,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:57,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:26:58,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:26:59,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:00,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:01,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:02,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:27:02,003 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:27:02,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:03,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:03,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:04,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:05,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:06,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:07,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:08,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:08,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:09,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:10,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:11,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:12,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:13,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:13,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:14,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:15,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:16,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:17,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:27:17,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:27:17,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:18,396 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:27:18,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:19,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:19,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:19,525 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:27:19,527 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:27:19,527 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:27:19,529 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:27:20,251 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:27:20,251 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:27:20,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:21,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:22,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:23,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:24,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:24,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:25,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:26,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:27,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:28,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:29,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:29,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:30,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:31,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:32,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:27:32,006 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:27:32,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:33,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:34,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:34,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:35,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:36,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:37,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:38,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:39,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:39,520 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:40,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:41,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:42,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:43,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:44,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:44,527 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:45,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:46,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:47,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:27:47,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:27:47,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:48,393 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:27:48,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:49,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:49,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:27:49,953 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:27:49,954 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:27:49,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:49,960 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:27:50,289 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:27:50,290 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:27:50,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:51,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:52,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:53,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:54,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:55,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:55,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:27:56,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:57,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:58,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:27:59,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:00,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:01,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:01,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:02,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:28:02,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:28:02,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:03,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:04,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:05,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:06,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:06,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:07,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:08,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:09,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:10,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:11,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:12,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:12,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:13,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:14,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:15,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:16,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:17,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:28:17,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:28:17,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:18,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:18,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:28:18,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:19,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:20,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:20,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:28:20,772 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:28:20,773 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:28:20,775 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:28:21,322 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:28:21,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:22,323 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:28:22,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:23,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:23,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:24,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:25,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:26,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:27,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:28,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:29,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:29,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:30,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:31,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:32,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:28:32,016 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:28:32,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:33,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:34,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:34,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:35,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:36,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:37,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:38,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:39,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:40,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:40,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:41,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:42,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:43,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:44,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:46,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:46,339 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:28:47,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:48,398 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:28:51,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:28:53,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:28:53,932 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:28:54,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:55,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:28:56,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:01,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:03,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:04,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:07,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:10,563 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:29:10,563 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:29:11,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:12,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:12,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:17,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:18,405 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:29:20,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:29:20,150 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:29:20,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:23,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:27,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:27,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:28,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:28,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:29,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 18:29:33,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:36,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:36,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:38,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:40,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:29:40,626 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:29:42,671 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:29:43,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:45,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:48,408 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:29:49,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:50,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:51,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:29:51,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:29:55,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:29:56,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:29:56,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:00,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:01,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:01,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:05,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:06,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:30:06,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:30:10,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:11,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:11,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:15,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:16,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:16,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:18,411 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:30:20,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:21,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:30:21,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:30:26,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:26,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:26,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:31,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:31,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:31,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:36,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:30:36,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:30:36,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:41,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:46,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:48,415 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:30:51,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:56,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:30:57,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:57,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:30:57,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:30:57,797 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:31:01,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:02,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:02,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:06,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:07,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:07,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:11,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:12,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:31:12,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:31:16,919 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:17,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:17,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:18,419 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:31:22,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:22,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:22,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:27,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:27,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:31:27,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:31:32,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:32,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:32,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:37,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:37,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:37,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:42,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:31:42,793 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:31:42,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:47,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:47,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:47,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:48,423 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:31:52,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:52,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:31:53,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:31:57,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:31:57,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:31:58,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:02,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:02,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:03,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:07,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:07,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:08,886 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:12,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:32:12,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:32:13,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:17,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:17,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:18,427 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:32:19,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:22,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:22,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:24,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:27,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:32:27,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:32:29,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:32,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:32,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:34,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:37,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:37,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:32:39,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:44,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:48,451 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:32:50,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:32:55,453 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:00,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:05,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:10,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:15,456 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:18,433 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:33:21,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:23,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:23,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:33:23,954 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:33:27,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:27,547 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:33:29,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:29,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:33,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:33,552 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:33:34,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:34,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:38,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:38,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:33:38,950 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:33:44,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:44,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:44,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:48,437 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:33:49,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:49,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:49,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:33:53,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:33:53,960 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:33:55,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:33:59,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:33:59,310 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:33:59,311 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:33:59,316 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:33:59,575 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:33:59,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:00,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:01,577 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:34:04,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:05,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:08,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:34:08,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:34:11,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:14,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:14,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:16,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:18,442 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:34:19,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:19,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:21,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:23,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:34:23,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:34:27,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:29,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:30,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:34:30,751 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:34:30,751 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:34:30,756 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:34:31,611 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:34:32,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:33,613 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:34:34,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:34,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:37,757 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:38,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:34:38,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:34:43,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:44,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:44,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:48,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:48,446 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:34:49,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:49,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:34:53,448 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:53,952 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:34:53,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:34:59,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:34:59,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:01,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:35:01,661 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:35:01,661 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:35:01,665 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:35:02,649 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:35:03,650 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:35:04,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:04,666 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:08,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:35:08,953 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:35:10,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:14,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:15,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:18,450 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:35:19,256 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:19,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:20,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:23,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:35:23,958 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:35:26,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:29,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:35:29,193 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:35:29,193 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:35:29,195 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:35:29,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:29,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:29,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:35:31,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:31,682 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:35:34,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:36,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:38,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:35:38,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:35:42,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:44,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:47,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:48,455 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:35:49,615 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:35:52,458 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:53,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:35:53,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:35:58,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:35:59,627 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:01,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:36:01,896 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:36:01,896 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:36:01,898 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:36:02,715 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:36:03,716 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:36:03,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:04,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:08,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:08,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:36:08,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:36:14,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:14,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:18,458 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:36:19,460 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:19,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:23,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:36:23,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:36:25,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:29,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:30,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:31,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:36:31,382 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:36:31,382 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:36:31,384 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:36:31,752 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:36:33,754 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:36:34,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:35,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:38,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:36:38,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:36:41,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:44,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:46,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:48,462 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:36:49,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:49,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:36:51,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:53,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:36:53,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:36:57,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:36:59,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:02,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:04,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:37:04,186 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:37:04,186 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:37:04,188 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:37:04,783 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:37:04,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:05,784 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:37:07,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:08,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:37:08,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:37:13,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:14,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:18,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:18,466 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:37:19,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:19,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:23,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:23,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:37:23,957 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:37:29,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:29,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:29,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:34,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:34,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:37,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:37:37,604 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:37:37,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:37:37,606 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:37:37,813 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:37:38,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:37:38,958 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:37:39,815 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:37:40,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:44,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:44,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:45,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:48,470 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:37:49,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:49,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:50,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:53,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:37:53,958 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:37:56,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:37:59,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:37:59,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:01,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:04,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:04,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:06,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:08,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:38:08,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:38:11,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:12,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:38:12,308 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:38:12,309 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:38:12,310 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:38:12,845 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:38:13,846 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:38:14,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:16,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:18,475 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:38:19,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:21,477 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:23,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:38:23,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:38:27,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:29,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:32,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:34,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:37,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:38,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:38:38,965 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:38:42,132 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:43,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:38:43,416 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:38:43,416 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:38:43,418 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:38:43,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:38:44,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:45,887 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:38:47,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:48,478 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:38:49,043 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:38:52,481 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:53,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:38:53,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:38:58,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:38:59,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:03,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:04,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:08,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:08,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:39:08,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:39:14,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:14,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:15,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:39:15,832 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:39:15,833 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:39:15,834 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:39:15,923 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:39:17,925 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:39:18,481 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:39:19,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:19,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:23,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:39:23,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:39:25,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:29,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:30,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:34,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:34,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:35,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:38,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:39:38,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:39:41,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:44,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:46,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:47,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:39:47,895 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:39:47,896 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:39:47,897 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:39:47,965 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:39:48,485 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:39:49,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:49,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:49,967 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:39:51,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:53,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:39:53,962 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:39:57,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:39:59,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:39:59,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:02,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:04,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:07,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:08,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:40:08,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:40:12,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:14,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:17,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:18,488 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:40:19,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:40:19,326 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:40:19,327 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:40:19,328 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:40:19,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:19,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:20,000 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:40:22,002 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:40:22,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:23,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:40:23,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:40:28,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:29,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:33,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:34,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:34,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:38,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:38,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:40:38,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:40:43,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:44,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:48,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:48,497 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:40:49,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:50,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:40:50,410 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:40:50,410 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:40:50,412 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:40:51,040 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:40:52,041 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:40:53,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:53,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:40:53,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:40:59,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:40:59,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:40:59,829 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:04,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:04,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:08,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:41:08,964 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:41:10,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:14,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:15,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:18,512 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:41:19,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:20,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:20,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:41:20,622 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:41:20,623 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:41:20,624 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:41:21,080 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:41:22,081 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:41:23,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:41:23,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:41:26,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:29,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:29,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:31,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:34,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:34,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:36,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:38,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:41:38,963 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:41:42,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:44,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:44,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:47,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:48,508 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:41:49,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:41:52,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:53,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:41:53,964 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:41:54,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:41:54,127 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:41:54,127 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:41:54,133 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:41:55,118 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:41:56,119 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:41:58,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:41:59,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:03,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:04,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:08,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:08,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:42:08,964 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:42:14,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:14,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:18,507 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:42:19,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:19,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:23,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:42:23,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:42:25,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:27,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:42:27,079 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:42:27,079 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:42:27,083 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:42:27,151 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:42:28,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:42:29,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:31,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:34,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:34,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:36,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:38,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:42:38,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:42:41,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:44,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:44,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:46,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:48,511 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:42:49,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:51,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:53,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:42:53,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:42:55,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:42:55,242 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:42:55,242 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:42:55,244 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:42:56,181 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:42:56,181 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:42:57,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:42:59,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:42:59,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:02,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:04,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:04,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:07,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:08,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:43:08,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:43:13,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:14,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:14,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:18,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:18,521 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:43:19,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:23,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:23,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:43:23,969 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:43:24,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:43:24,701 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:43:24,702 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:43:24,703 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:43:25,213 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:43:26,214 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:43:28,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:29,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:33,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:34,673 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:38,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:38,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:43:38,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:43:44,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:44,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:48,521 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:43:49,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:49,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:53,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:43:53,977 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:43:55,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:43:57,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:43:57,234 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:43:57,234 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:43:57,239 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:43:57,244 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:43:58,246 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:43:59,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:43:59,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:00,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:04,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:04,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:05,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:08,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:44:08,971 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:44:11,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:14,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:14,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:16,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:18,524 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:44:19,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:19,518 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:21,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:23,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:44:23,981 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:44:27,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:27,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:44:27,211 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:44:27,211 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:44:27,213 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:44:27,278 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:44:28,279 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:44:29,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:32,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:34,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:34,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:37,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:38,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:44:38,973 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:44:43,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:44,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:48,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:48,528 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:44:49,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:49,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:44:53,531 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:53,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:44:53,973 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:44:59,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:44:59,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:02,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:45:02,095 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:45:02,095 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:45:02,098 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:45:02,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:45:04,317 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:45:04,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:04,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:05,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:08,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:45:08,973 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:45:10,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:14,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:14,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:15,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:18,532 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:45:19,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:19,587 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:20,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:23,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:45:23,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:45:26,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:29,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:45:29,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:45:29,344 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:45:29,345 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:45:29,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:29,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:30,346 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:45:30,347 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:45:31,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:34,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:36,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:38,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:45:38,975 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:45:42,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:44,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:47,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:48,536 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:45:49,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:49,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:52,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:53,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:45:53,980 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:45:57,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:45:57,188 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:45:57,189 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:45:57,190 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:45:57,374 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:45:58,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:45:58,375 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:45:59,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:45:59,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:03,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:04,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:04,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:08,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:08,977 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:46:08,978 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:46:14,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:14,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:14,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:18,541 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:46:19,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:19,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:19,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:23,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:46:23,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:46:25,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:26,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:46:26,899 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:46:26,900 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:46:26,901 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:46:27,406 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:46:28,407 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:46:29,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:29,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:30,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:34,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:34,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:35,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:38,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:46:38,982 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:46:41,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:44,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:44,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:46,063 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:48,545 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:46:49,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:49,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:51,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:53,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:46:53,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:46:57,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:46:59,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:59,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:46:59,700 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:46:59,701 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:46:59,701 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:46:59,703 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:47:00,442 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:47:02,443 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:47:02,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:04,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:07,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:08,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:47:08,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:47:13,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:14,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:18,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:18,549 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:47:19,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:23,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:23,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:47:23,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:47:29,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:29,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:30,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:47:30,425 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:47:30,425 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:47:30,427 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:47:30,473 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:47:32,476 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:47:34,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:34,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:38,983 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:47:38,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:47:40,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:44,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:44,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:45,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:48,553 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:47:49,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:49,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:47:50,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:53,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:47:53,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:47:56,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:47:59,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:01,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:03,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:48:03,193 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:48:03,193 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:48:03,195 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:48:03,504 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:48:04,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:04,505 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:48:06,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:08,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:48:08,984 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:48:12,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:14,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:14,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:17,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:18,558 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:48:19,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:22,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:23,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:48:23,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:48:28,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:29,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:33,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:34,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:34,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:48:34,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:48:34,678 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:48:34,679 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:48:35,538 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:48:36,540 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:48:38,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:38,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:48:38,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:48:44,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:44,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:48,562 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:48:49,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:48:49,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:53,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:48:53,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:48:55,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:48:59,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:00,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:04,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:04,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:49:04,443 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:49:04,443 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:49:04,445 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:49:04,574 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:49:05,446 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:06,576 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:49:08,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:49:08,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:49:11,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:14,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:16,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:18,567 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:49:19,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:21,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:23,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:49:23,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:49:27,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:29,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:32,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:34,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:35,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:49:35,407 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:49:35,408 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:49:35,410 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:49:35,605 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:49:36,606 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:49:37,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:38,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:49:38,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:49:43,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:44,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:48,147 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:48,570 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:49:49,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:53,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:53,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:49:53,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:49:59,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:49:59,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:49:59,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:04,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:04,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:50:04,175 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:50:04,176 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:50:04,178 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:50:04,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:04,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:04,636 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:50:06,638 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:50:08,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:50:08,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:50:10,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:14,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:15,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:18,574 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:50:19,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:19,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:20,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:23,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:50:23,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:50:26,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:29,626 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:31,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:31,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:50:31,861 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:50:31,861 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:50:31,862 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:50:32,671 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:50:32,671 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:50:34,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:36,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:38,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:50:38,985 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:50:42,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:44,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:47,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:48,577 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:50:49,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:50:52,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:53,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:50:53,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:50:58,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:50:59,635 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:03,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:04,639 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:05,689 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:51:05,689 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:51:05,690 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:51:05,691 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:51:05,704 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:51:06,705 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:51:08,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:08,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:51:08,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:51:14,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:14,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:18,581 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:51:19,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:19,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:23,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:51:23,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:51:25,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:29,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:30,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:33,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:51:33,660 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:51:33,661 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:51:33,663 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:51:33,738 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:51:34,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:34,738 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:51:35,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:38,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:51:38,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:51:41,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:44,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:46,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:48,586 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:51:49,647 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:51:51,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:53,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:51:53,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:51:57,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:51:59,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:02,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:03,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:52:03,476 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:52:03,476 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:52:03,478 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:52:03,767 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:52:04,768 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:52:04,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:04,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:07,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:08,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:52:08,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:52:13,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:14,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:14,895 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:18,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:18,589 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:52:19,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:19,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:23,592 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:23,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:52:23,986 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:52:29,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:29,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:34,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:34,953 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:35,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:52:35,261 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:52:35,262 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:52:35,263 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:52:35,794 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:52:36,795 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:52:38,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:52:38,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:52:39,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:44,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:44,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:48,594 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:52:49,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:49,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:52:53,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:52:53,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:52:55,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:52:59,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:00,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:04,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:05,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:06,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:53:06,179 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:53:06,180 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:53:06,181 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:53:06,827 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:53:08,830 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:53:08,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:53:08,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:53:10,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:14,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:15,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:18,598 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:53:19,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:20,600 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:23,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:53:23,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:53:26,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:29,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:31,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:34,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:36,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:37,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:53:37,103 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:53:37,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:53:37,105 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:53:37,862 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:53:38,863 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:53:38,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:53:38,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:53:41,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:44,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:44,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:46,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:48,602 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:53:49,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:49,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:51,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:53,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:53:53,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:53:57,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:53:59,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:53:59,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:02,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:04,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:07,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:08,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:54:08,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:54:09,195 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:54:09,198 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:54:09,198 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:54:09,200 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:54:09,897 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:54:10,898 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:54:12,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:13,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:17,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:18,605 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:54:18,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:22,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:23,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:54:23,987 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:54:28,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:28,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:33,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:34,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:38,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:38,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:54:38,996 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:54:39,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:54:39,438 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:54:39,438 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:54:39,440 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:54:39,928 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:54:40,929 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:54:43,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:44,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:48,442 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:48,608 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:54:49,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:53,610 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:54:53,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:54:53,992 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:54:59,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:54:59,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:04,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:04,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:08,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:55:08,998 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:55:10,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:11,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:55:11,104 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:55:11,105 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:55:11,107 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:55:11,963 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:55:12,964 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:55:14,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:15,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:18,611 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:55:19,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:20,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:23,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:55:23,994 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:55:26,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:29,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:31,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:34,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:36,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:38,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:55:38,997 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:55:41,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:42,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:55:42,037 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:55:42,038 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:55:42,039 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:55:43,001 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:55:43,001 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:55:44,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:47,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:48,615 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:55:49,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:52,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:53,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:55:53,995 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:55:58,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:55:59,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:55:59,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:03,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:04,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:04,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:08,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:09,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:56:09,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:56:11,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:56:11,401 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:56:11,402 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:56:11,403 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:56:12,032 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:56:13,033 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:56:13,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:14,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:18,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:18,618 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:56:19,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:23,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:23,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:56:24,000 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:56:29,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:29,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:34,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:34,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:38,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:56:39,000 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:56:39,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:56:39,437 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:56:39,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:39,437 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:56:39,440 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:56:40,073 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:56:41,074 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:56:44,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:44,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:48,622 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:56:49,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:56:49,625 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:54,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:56:54,000 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:56:55,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:56:59,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:00,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:04,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:05,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:09,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:57:09,006 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:57:11,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:11,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:57:11,294 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:57:11,294 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:57:11,296 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:57:12,113 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:57:13,115 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:57:14,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:16,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:18,626 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:57:19,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:21,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:24,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:57:24,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:57:27,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:29,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:32,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:34,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:37,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:39,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:57:39,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:57:41,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:57:41,373 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:57:41,373 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:57:41,374 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:57:42,144 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:57:42,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:43,144 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:57:44,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:47,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:48,630 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:57:49,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:57:52,633 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:54,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:57:54,008 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:57:58,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:57:59,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:03,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:04,218 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:08,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:09,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:58:09,014 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:58:12,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:58:12,464 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:58:12,465 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:58:12,466 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:58:13,169 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:58:13,169 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:58:13,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:14,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:18,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:18,634 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:58:19,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:23,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:24,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:58:24,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:58:29,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:29,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:29,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:34,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:34,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:34,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:39,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:58:39,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:58:39,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:44,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:44,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:46,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:58:46,492 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:58:46,492 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:58:46,494 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:58:47,201 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:58:47,202 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:58:48,639 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:58:49,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:49,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:49,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:58:54,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:58:54,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:58:55,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:58:59,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:00,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:04,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:04,646 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:05,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:09,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:59:09,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:59:10,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:14,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:15,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:16,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:59:16,976 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:59:16,977 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:59:16,979 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:59:17,239 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:59:18,643 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:59:19,241 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:59:19,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:20,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:24,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:59:24,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:59:26,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:29,692 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:29,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:31,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:34,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:34,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:36,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:39,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:59:39,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:59:41,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:44,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:45,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 18:59:45,876 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 18:59:45,877 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 18:59:45,878 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 18:59:46,266 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 18:59:46,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:47,267 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 18:59:48,647 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 18:59:49,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:49,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 18:59:52,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:54,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 18:59:54,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 18:59:58,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 18:59:59,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:03,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:04,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:04,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:08,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:09,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:00:09,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:00:14,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:14,714 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:16,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:00:16,216 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:00:16,216 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:00:16,217 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:00:16,292 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:00:17,293 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:00:18,650 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:00:19,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:19,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:24,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:00:24,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:00:25,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:29,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:30,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:34,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:35,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:39,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:00:39,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:00:40,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:44,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:00:44,108 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:00:44,109 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:00:44,110 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:00:44,319 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:00:44,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:45,320 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:00:46,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:48,654 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:00:49,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:00:51,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:54,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:00:54,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:00:57,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:00:59,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:02,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:04,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:04,726 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:07,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:09,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:01:09,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:01:12,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:14,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:16,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:01:16,471 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:01:16,471 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:01:16,472 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:01:17,356 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:01:17,357 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:01:17,473 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:18,658 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:01:19,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:19,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:22,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:24,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:01:24,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:01:28,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:29,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:29,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:33,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:34,945 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:34,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:38,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:39,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:01:39,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:01:44,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:44,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:47,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:01:47,282 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:01:47,283 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:01:47,284 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:01:47,381 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:01:48,662 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:01:49,382 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:01:49,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:49,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:49,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:54,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:01:54,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:01:55,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:01:59,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:01:59,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:00,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:04,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:04,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:05,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:09,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:02:09,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:02:11,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:14,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:16,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:18,180 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:02:18,181 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:02:18,181 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:02:18,183 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:02:18,414 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:02:18,666 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:02:19,419 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:02:19,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:21,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:24,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:02:24,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:02:27,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:29,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:32,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:34,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:37,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:39,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:02:39,011 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:02:42,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:44,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:47,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:48,674 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:02:49,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:02:50,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:02:50,932 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:02:50,932 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:02:50,934 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:02:51,455 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:02:52,456 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:02:52,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:54,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:02:54,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:02:58,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:02:59,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:03,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:04,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:04,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:08,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:09,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:03:09,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:03:13,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:14,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:18,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:18,681 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:03:19,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:22,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:03:22,855 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:03:22,856 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:03:22,857 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:03:23,493 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:03:23,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:24,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:03:24,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:03:24,494 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:03:29,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:29,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:29,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:34,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:34,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:34,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:39,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:03:39,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:03:39,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:44,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:44,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:44,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:48,686 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:03:49,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:49,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:52,965 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:03:52,967 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:03:52,967 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:03:52,971 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:03:53,526 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:03:54,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:03:54,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:03:54,527 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:03:55,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:03:59,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:03:59,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:00,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:04,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:04,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:05,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:09,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:04:09,012 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:04:11,114 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:14,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:14,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:16,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:18,690 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:04:19,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:21,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:23,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:04:23,940 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:04:23,941 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:04:23,944 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:04:24,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:04:24,372 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:04:24,560 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:04:26,562 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:04:27,475 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:29,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:32,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:34,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:34,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:37,476 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:39,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:04:39,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:04:43,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:44,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:44,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:48,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:48,694 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:04:49,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:04:52,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:04:52,666 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:04:52,666 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:04:52,667 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:04:53,595 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:04:53,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:54,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:04:54,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:04:54,596 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:04:59,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:04:59,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:04,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:04,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:09,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:05:09,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:05:10,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:14,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:15,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:18,689 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:05:19,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:20,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:22,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:05:22,798 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:05:22,799 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:05:22,800 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:05:23,629 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:05:24,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:05:24,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:05:24,630 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:05:26,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:29,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:31,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:34,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:36,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:39,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:05:39,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:05:42,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:44,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:47,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:48,700 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:05:49,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:52,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:05:52,158 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:05:52,159 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:05:52,161 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:05:52,658 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:05:53,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:54,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:05:54,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:05:54,660 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:05:58,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:05:59,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:05:59,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:03,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:04,571 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:08,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:09,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:06:09,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:06:14,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:14,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:14,573 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:18,702 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:06:19,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:19,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:23,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:06:23,083 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:06:23,084 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:06:23,089 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:06:23,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:06:24,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:06:24,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:06:24,691 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:06:25,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:29,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:30,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:34,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:35,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:39,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:06:39,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:06:40,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:44,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:44,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:45,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:48,706 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:06:49,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:49,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:50,716 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:54,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:06:54,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:06:56,200 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:06:56,739 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:06:56,740 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:06:56,741 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:06:56,744 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:06:57,726 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:06:58,727 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:06:59,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:06:59,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:01,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:04,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:04,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:06,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:09,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:07:09,090 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:07:12,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:14,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:17,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:18,711 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:07:19,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:22,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:24,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:07:24,090 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:07:28,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:29,552 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:07:29,554 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:07:29,555 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:07:29,560 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:07:29,769 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:07:29,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:29,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:30,770 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:07:33,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:34,955 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:38,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:39,089 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:07:39,090 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:07:44,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:44,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:44,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:48,715 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:07:49,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:49,957 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:07:54,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:07:54,107 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:07:55,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:07:58,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:07:58,560 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:07:58,561 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:07:58,563 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:07:58,793 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:07:59,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:00,564 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:00,796 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:08:04,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:05,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:09,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:08:09,094 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:08:11,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:14,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:16,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:18,720 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:08:19,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:21,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:24,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:08:24,111 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:08:27,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:29,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:30,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:08:30,438 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:08:30,439 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:08:30,442 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:08:30,836 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:08:32,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:32,838 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:08:34,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:34,985 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:37,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:39,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:08:39,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:08:43,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:44,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:44,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:48,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:48,723 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:08:49,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:49,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:53,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:08:54,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:08:54,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:08:58,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:08:58,689 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:08:58,690 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:08:58,692 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:08:58,879 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:08:59,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:59,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:08:59,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:00,881 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:09:04,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:04,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:09,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:09:09,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:09:10,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:14,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:15,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:18,727 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:09:19,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:20,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:24,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:09:24,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:09:26,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:29,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:31,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:09:31,054 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:09:31,055 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:09:31,057 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:09:31,920 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:09:32,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:32,921 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:09:34,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:37,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:39,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:09:39,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:09:42,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:44,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:44,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:47,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:48,731 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:09:49,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:09:52,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:54,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:09:54,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:09:58,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:09:59,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:00,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:10:00,797 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:10:00,798 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:10:00,799 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:10:00,954 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:10:02,955 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:10:03,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:04,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:08,801 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:09,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:10:09,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:10:14,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:14,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:18,735 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:10:19,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:19,737 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:24,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:10:24,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:10:25,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:29,228 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:30,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:31,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:10:31,605 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:10:31,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:10:31,607 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:10:32,001 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:10:33,002 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:10:34,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:35,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:39,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:10:39,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:10:41,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:44,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:46,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:48,738 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:10:49,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:10:51,740 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:54,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:10:54,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:10:57,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:10:59,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:02,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:03,723 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:11:03,724 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:11:03,724 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:11:03,726 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:11:04,027 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:11:04,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:05,028 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:11:07,727 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:09,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:11:09,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:11:13,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:14,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:18,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:18,742 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:11:19,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:23,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:24,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:11:24,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:11:29,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:29,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:34,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:34,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:36,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:11:36,262 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:11:36,262 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:11:36,264 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:11:37,068 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:11:39,070 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:11:39,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:11:39,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:11:40,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:44,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:45,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:48,745 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:11:49,324 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:11:50,747 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:54,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:11:54,099 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:11:56,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:11:59,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:01,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:04,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:06,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:12:06,074 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:12:06,075 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:12:06,077 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:12:06,095 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:12:07,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:07,096 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:12:09,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:12:09,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:12:12,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:14,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:14,484 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:17,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:18,749 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:12:19,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:19,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:22,751 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:24,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:12:24,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:12:28,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:29,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:29,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:33,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:34,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:37,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:12:37,089 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:12:37,090 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:12:37,091 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:12:37,127 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:12:39,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:39,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:12:39,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:12:39,129 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:12:44,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:44,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:48,753 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:12:49,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:12:49,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:54,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:12:54,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:12:55,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:12:59,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:00,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:04,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:05,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:08,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:13:08,186 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:13:08,187 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:13:08,188 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:13:09,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:13:09,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:13:09,161 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:13:09,162 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:13:11,230 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:14,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:16,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:18,757 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:13:19,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:19,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:21,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:24,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:13:24,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:13:27,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:29,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:32,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:34,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:36,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:13:36,780 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:13:36,781 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:13:36,782 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:13:37,189 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:13:37,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:39,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:13:39,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:13:39,191 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:13:43,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:44,613 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:48,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:48,762 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:13:49,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:49,614 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:53,764 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:54,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:13:54,100 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:13:59,188 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:13:59,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:13:59,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:04,196 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:04,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:08,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:14:08,255 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:14:08,256 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:14:08,257 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:14:09,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:14:09,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:14:09,225 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:14:09,226 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:14:09,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:14,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:14,693 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:18,765 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:14:19,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:19,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:24,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:14:24,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:14:25,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:29,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:30,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:34,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:35,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:38,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:14:38,977 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:14:38,977 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:14:38,979 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:14:39,255 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:14:39,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:14:39,394 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:14:40,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:41,257 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:14:44,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:44,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:45,555 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:48,769 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:14:49,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:49,706 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:50,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:54,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:14:54,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:14:56,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:14:59,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:14:59,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:01,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:04,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:06,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:08,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:15:08,497 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:15:08,498 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:15:08,500 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:15:09,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:15:09,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:15:09,284 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:15:09,285 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:15:12,201 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:14,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:14,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:17,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:18,773 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:15:19,907 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:19,908 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:22,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:24,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:15:24,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:15:28,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:29,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:29,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:33,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:34,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:34,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:38,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:39,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:15:39,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:15:41,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:15:41,774 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:15:41,774 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:15:41,776 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:15:42,316 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:15:43,316 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:15:43,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:44,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:48,777 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:15:48,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:49,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:53,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:15:54,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:15:54,129 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:15:59,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:15:59,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:04,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:04,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:09,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:16:09,132 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:16:10,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:16:10,170 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:16:10,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:16:10,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:10,172 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:16:10,344 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:16:11,345 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:16:14,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:15,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:18,781 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:16:19,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:20,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:24,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:16:24,132 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:16:26,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:29,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:31,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:34,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:36,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:39,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:16:39,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:16:41,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:41,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:16:41,677 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:16:41,677 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:16:41,679 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:16:42,390 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:16:43,391 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:16:44,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:46,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:48,785 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:16:49,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:16:51,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:54,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:16:54,135 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:16:57,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:16:59,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:02,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:04,190 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:07,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:09,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:17:09,142 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:17:12,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:14,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:14,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:17:14,537 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:17:14,538 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:17:14,539 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:17:15,429 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:17:15,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:17:17,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:18,789 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:17:19,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:22,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:24,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:17:24,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:17:28,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:29,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:33,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:34,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:38,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:39,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:17:39,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:17:44,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:44,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:46,516 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:17:46,518 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:17:46,519 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:17:46,520 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:17:47,461 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:17:47,461 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:17:48,793 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:17:49,208 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:17:49,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:54,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:17:54,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:17:55,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:17:59,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:00,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:04,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:05,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:09,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:18:09,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:18:11,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:14,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:16,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:18,798 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:18:19,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:19,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:18:19,926 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:18:19,926 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:18:19,928 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:18:20,488 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:18:21,489 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:18:21,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:24,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:18:24,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:18:27,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:29,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:32,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:34,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:37,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:39,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:18:39,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:18:42,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:44,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:47,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:48,802 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:18:49,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:18:50,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:18:50,845 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:18:50,845 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:18:50,847 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:18:51,518 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:18:52,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:53,520 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:18:54,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:18:54,138 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:18:58,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:18:59,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:03,298 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:04,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:08,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:09,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:19:09,139 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:19:14,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:14,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:18,803 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:19:19,273 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:19,812 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:24,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:19:24,077 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:19:24,077 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:19:24,079 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:19:24,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:24,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:19:24,494 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:19:24,555 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:19:25,556 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:19:25,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:29,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:30,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:34,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:35,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:39,221 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:19:39,222 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:19:41,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:44,496 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:46,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:48,807 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:19:49,497 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:19:51,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:54,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:19:54,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:19:56,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:19:56,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:19:56,383 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:19:56,385 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:19:56,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:19:57,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:19:57,585 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:19:59,512 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:02,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:04,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:07,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:09,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:20:09,225 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:20:13,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:14,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:18,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:18,811 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:20:19,588 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:23,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:24,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:20:24,231 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:20:29,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:29,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:29,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:20:29,791 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:20:29,792 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:20:29,793 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:20:30,632 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:20:31,634 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:20:34,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:34,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:39,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:20:39,225 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:20:40,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:44,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:45,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:48,815 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:20:49,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:50,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:54,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:20:54,225 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:20:56,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:20:59,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:20:59,885 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:20:59,886 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:20:59,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:20:59,888 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:21:00,679 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:21:01,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:21:01,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:04,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:06,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:09,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:21:09,225 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:21:12,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:14,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:14,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:17,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:18,819 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:21:19,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:19,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:23,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:24,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:21:24,225 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:21:29,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:29,624 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:31,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:21:31,722 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:21:31,722 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:21:31,724 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:21:32,721 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:21:33,722 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:21:34,621 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:34,725 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:39,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:21:39,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:21:40,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:44,622 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:45,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:48,824 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:21:49,623 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:21:50,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:54,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:21:54,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:21:56,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:21:59,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:01,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:04,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:22:04,506 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:22:04,507 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:22:04,508 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:22:04,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:04,756 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:22:04,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:05,758 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:22:06,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:09,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:22:09,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:22:12,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:14,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:14,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:17,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:18,827 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:22:19,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:19,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:22,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:24,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:22:24,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:22:28,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:29,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:29,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:33,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:34,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:22:34,353 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:22:34,353 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:22:34,355 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:22:34,787 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:22:34,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:34,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:35,788 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:22:39,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:22:39,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:22:39,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:44,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:44,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:48,831 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:22:49,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:49,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:22:54,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:22:54,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:22:55,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:22:59,992 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:00,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:04,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:05,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:06,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:23:06,014 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:23:06,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:23:06,016 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:23:06,819 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:23:07,820 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:23:09,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:23:09,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:23:10,363 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:14,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:15,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:18,835 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:23:19,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:20,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:24,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:23:24,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:23:26,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:29,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:31,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:34,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:35,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:23:35,356 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:23:35,357 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:23:35,359 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:23:35,845 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:23:36,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:37,847 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:23:39,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:23:39,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:23:41,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:44,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:44,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:46,387 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:48,838 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:23:49,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:49,999 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:23:51,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:23:54,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:23:54,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:23:57,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:00,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:02,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:05,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:05,956 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:24:05,958 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:24:05,958 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:24:05,959 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:24:06,880 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:24:07,881 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:24:07,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:09,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:24:09,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:24:13,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:15,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:15,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:18,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:18,841 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:24:20,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:20,112 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:23,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:24,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:24:24,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:24:29,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:30,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:30,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:34,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:35,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:35,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:38,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:24:38,619 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:24:38,620 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:24:38,621 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:24:38,911 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:24:39,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:24:39,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:24:39,912 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:24:40,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:45,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:45,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:45,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:48,847 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:24:50,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:50,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:24:50,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:24:54,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:24:54,227 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:24:56,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:00,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:00,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:01,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:05,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:06,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:09,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:25:09,229 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:25:10,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:25:10,859 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:25:10,860 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:25:10,861 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:25:10,947 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:25:11,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:11,948 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:25:14,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:14,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:16,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:18,850 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:25:19,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:19,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:22,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:24,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:25:24,230 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:25:28,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:29,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:29,271 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:33,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:34,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:34,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:38,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:39,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:25:39,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:25:44,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:44,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:44,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:25:44,817 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:25:44,817 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:25:44,819 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:25:44,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:25:45,983 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:25:48,854 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:25:49,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:49,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:25:49,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:54,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:25:54,233 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:25:55,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:25:59,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:00,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:04,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:05,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:09,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:26:09,233 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:26:11,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:14,443 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:15,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:26:15,581 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:26:15,582 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:26:15,583 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:26:16,018 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:26:16,584 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:18,019 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:26:18,856 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:26:19,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:21,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:24,232 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:26:24,233 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:26:27,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:29,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:32,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:34,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:37,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:39,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:26:39,236 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:26:42,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:44,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:26:44,247 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:26:44,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:26:44,248 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:26:44,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:45,046 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:26:46,047 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:26:48,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:48,860 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:26:49,660 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:26:53,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:54,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:26:54,237 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:26:59,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:26:59,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:04,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:04,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:09,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:27:09,242 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:27:09,392 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:13,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:27:13,972 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:27:13,972 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:27:13,974 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:27:14,079 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:27:14,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:14,974 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:16,081 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:27:18,865 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:27:19,672 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:20,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:24,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:27:24,240 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:27:26,335 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:29,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:31,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:34,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:36,337 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:39,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:27:39,259 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:27:41,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:44,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:45,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:27:45,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:27:45,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:27:45,087 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:27:45,113 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:27:46,114 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:27:47,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:48,868 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:27:49,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:27:52,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:54,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:27:54,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:27:58,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:27:59,870 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:03,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:04,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:08,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:09,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:28:09,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:28:14,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:14,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:15,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:28:15,874 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:28:15,874 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:28:15,875 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:28:16,154 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:28:18,156 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:28:18,872 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:28:19,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:20,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:24,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:28:24,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:28:25,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:29,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:30,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:34,288 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:35,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:39,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:28:39,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:28:40,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:44,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:45,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:45,875 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:28:45,877 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:28:45,878 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:28:45,880 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:28:46,191 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:28:48,193 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:28:48,876 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:28:49,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:28:50,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:54,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:28:54,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:28:56,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:28:59,289 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:01,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:04,290 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:06,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:09,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:29:09,245 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:29:11,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:14,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:16,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:18,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:29:18,152 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:29:18,152 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:29:18,154 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:29:18,228 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:29:18,881 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:29:19,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:19,306 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:20,230 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:29:21,883 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:24,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:29:24,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:29:27,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:29,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:32,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:34,308 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:37,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:39,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:29:39,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:29:43,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:44,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:48,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:48,883 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:29:49,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:49,509 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:29:49,511 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:29:49,512 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:29:49,513 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:29:50,265 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:29:50,266 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:29:53,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:29:54,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:29:54,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:29:59,315 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:29:59,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:04,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:04,395 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:09,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:30:09,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:30:10,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:14,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:15,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:18,889 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:30:19,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:20,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:30:20,016 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:30:20,017 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:30:20,018 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:30:20,295 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:30:21,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:22,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:30:24,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:30:24,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:30:26,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:29,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:31,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:34,431 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:34,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:36,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:39,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:30:39,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:30:42,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:44,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:44,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:47,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:48,893 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:30:49,436 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:30:50,570 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:30:50,571 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:30:50,572 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:30:50,573 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:30:51,330 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:30:52,331 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:30:52,574 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:54,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:30:54,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:30:58,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:30:59,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:03,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:04,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:08,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:09,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:31:09,246 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:31:13,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:14,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:18,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:18,898 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:31:19,452 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:20,695 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:31:20,696 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:31:20,696 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:31:20,698 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:31:21,366 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:31:22,368 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:31:23,699 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:24,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:31:24,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:31:29,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:29,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:34,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:34,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:34,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:39,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:31:39,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:31:39,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:44,364 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:44,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:48,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:31:48,448 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:31:48,449 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:31:48,450 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:31:48,900 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:31:49,398 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:31:49,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:49,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:49,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:50,399 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:31:54,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:31:54,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:31:55,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:31:59,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:31:59,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:00,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:04,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:04,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:05,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:09,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:32:09,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:32:10,414 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:14,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:15,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:17,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:32:17,807 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:32:17,807 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:32:17,809 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:32:18,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:32:18,431 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:32:18,903 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:32:19,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:19,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:20,906 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:24,246 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:32:24,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:32:26,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:29,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:29,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:31,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:34,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:34,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:36,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:39,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:32:39,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:32:42,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:44,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:47,086 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:32:47,088 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:32:47,088 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:32:47,090 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:32:47,463 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:32:48,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:48,465 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:32:48,908 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:32:49,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:49,629 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:53,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:54,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:32:54,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:32:59,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:32:59,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:32:59,631 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:04,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:04,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:04,632 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:09,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:33:09,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:33:09,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:14,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:14,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:18,910 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:33:19,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:33:19,467 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:33:19,467 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:33:19,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:19,469 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:33:19,502 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:33:19,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:19,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:20,503 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:33:24,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:33:24,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:33:25,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:29,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:30,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:34,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:35,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:39,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:33:39,247 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:33:41,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:44,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:46,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:48,917 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:33:49,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:33:50,884 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:33:50,885 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:33:50,886 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:33:50,887 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:33:51,533 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:33:51,888 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:33:52,534 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:33:54,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:33:54,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:33:57,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:00,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:02,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:05,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:07,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:09,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:34:09,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:34:12,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:15,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:17,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:18,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:34:18,318 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:34:18,319 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:34:18,320 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:34:18,563 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:34:18,922 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:34:20,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:21,566 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:34:22,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:24,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:34:24,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:34:28,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:30,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:33,405 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:35,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:35,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:38,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:39,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:34:39,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:34:44,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:45,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:48,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:34:48,836 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:34:48,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:34:48,838 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:34:48,924 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:34:49,594 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:34:49,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:50,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:50,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:34:51,596 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:34:54,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:34:54,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:34:55,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:34:59,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:00,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:04,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:04,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:05,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:09,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:35:09,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:35:10,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:14,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:14,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:15,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:18,930 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:35:19,301 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:19,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:35:19,430 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:35:19,430 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:35:19,432 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:35:19,629 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:35:20,432 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:21,631 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:35:24,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:35:24,248 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:35:26,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:29,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:31,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:34,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:34,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:36,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:39,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:35:39,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:35:42,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:44,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:44,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:47,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:48,933 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:35:49,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:35:51,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:35:51,362 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:35:51,363 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:35:51,364 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:35:51,658 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:35:52,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:53,659 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:35:54,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:35:54,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:35:58,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:35:59,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:03,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:04,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:08,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:09,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:36:09,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:36:14,316 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:14,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:18,938 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:36:19,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:19,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:20,441 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:36:20,442 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:36:20,442 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:36:20,444 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:36:20,682 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:36:23,684 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:36:24,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:36:24,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:36:25,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:29,359 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:30,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:34,360 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:35,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:39,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:36:39,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:36:41,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:44,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:46,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:48,942 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:36:49,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:51,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:52,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:36:52,122 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:36:52,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:36:52,124 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:36:52,724 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:36:53,725 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:36:54,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:36:54,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:36:57,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:36:59,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:36:59,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:02,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:04,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:04,541 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:07,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:09,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:37:09,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:37:13,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:14,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:14,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:18,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:18,943 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:37:19,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:21,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:37:21,590 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:37:21,591 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:37:21,592 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:37:21,760 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:37:23,593 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:23,762 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:37:24,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:37:24,249 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:37:29,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:29,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:34,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:34,561 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:34,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:39,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:37:39,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:37:39,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:44,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:44,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:48,945 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:37:49,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:49,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:52,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:37:52,260 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:37:52,261 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:37:52,263 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:37:52,793 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:37:53,794 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:37:54,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:37:54,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:37:55,406 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:37:59,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:37:59,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:00,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:04,668 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:04,669 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:05,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:09,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:38:09,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:38:11,325 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:14,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:14,670 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:16,334 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:18,961 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:38:19,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:21,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:38:21,892 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:38:21,893 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:38:21,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:21,894 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:38:22,829 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:38:23,830 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:38:24,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:38:24,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:38:27,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:29,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:32,371 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:34,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:37,372 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:39,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:38:39,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:38:42,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:44,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:47,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:48,955 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:38:49,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:52,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:54,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:38:54,251 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:38:54,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:38:54,404 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:38:54,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:38:54,409 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:38:54,868 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:38:55,869 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:38:58,410 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:38:59,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:38:59,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:03,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:04,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:04,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:08,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:09,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:39:09,250 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:39:14,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:14,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:14,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:18,965 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:39:19,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:19,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:24,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:39:24,253 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:39:25,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:39:25,072 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:39:25,073 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:39:25,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:25,079 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:39:25,901 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:39:27,903 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:39:29,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:29,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:30,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:34,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:34,831 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:35,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:39,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:39:39,254 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:39:40,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:44,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:44,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:45,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:48,961 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:39:49,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:50,962 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:53,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:39:53,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:39:53,087 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:39:53,088 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:39:53,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:39:53,945 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:39:54,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:39:54,254 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:39:56,420 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:39:59,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:39:59,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:01,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:04,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:04,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:06,421 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:09,253 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:40:09,254 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:40:12,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:14,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:14,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:17,328 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:18,975 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:40:19,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:19,850 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:22,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:40:22,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:40:22,344 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:40:22,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:22,345 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:40:22,982 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:40:23,983 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:40:24,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:40:24,260 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:40:27,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:29,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:29,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:32,437 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:34,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:34,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:37,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:39,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:40:39,259 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:40:43,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:44,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:44,856 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:48,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:48,969 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:40:49,863 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:53,976 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:54,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:40:54,220 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:40:54,220 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:40:54,225 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:40:54,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:40:54,641 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:40:55,012 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:40:56,013 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:40:59,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:40:59,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:40:59,864 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:04,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:04,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:04,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:09,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:41:09,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:41:10,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:14,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:15,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:18,974 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:41:19,873 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:20,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:24,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:41:24,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:41:24,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:41:24,765 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:41:24,765 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:41:24,768 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:41:25,049 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:41:26,051 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:41:26,769 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:29,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:29,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:31,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:34,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:34,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:36,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:39,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:41:39,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:41:42,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:44,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:44,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:47,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:48,978 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:41:49,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:49,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:41:52,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:41:54,636 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:41:54,636 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:41:56,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:41:56,409 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:41:56,410 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:41:56,415 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:41:57,082 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:41:58,083 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:41:58,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:00,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:00,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:03,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:05,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:05,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:08,417 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:09,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:42:09,637 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:42:13,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:15,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:15,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:18,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:18,982 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:42:20,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:20,607 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:23,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:24,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:42:24,640 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:42:26,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:42:26,905 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:42:26,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:42:26,910 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:42:27,112 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:42:28,113 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:42:29,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:30,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:30,127 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:34,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:35,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:35,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:39,638 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:42:39,639 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:42:40,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:45,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:45,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:45,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:48,986 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:42:50,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:50,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:42:50,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:54,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:42:54,649 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:42:56,731 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:42:59,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:42:59,350 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:42:59,350 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:42:59,355 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:43:00,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:00,152 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:43:00,153 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:43:02,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:05,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:05,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:07,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:09,643 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:43:09,644 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:43:12,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:15,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:15,140 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:17,715 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:18,991 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:43:20,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:20,141 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:22,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:24,652 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:43:24,652 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:43:28,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:29,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:43:29,541 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:43:29,542 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:43:29,545 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:43:30,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:30,189 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:43:32,191 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:43:34,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:35,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:39,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:39,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:43:39,650 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:43:44,815 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:45,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:48,995 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:43:49,997 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:50,152 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:43:54,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:43:54,656 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:43:55,768 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:43:57,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:43:57,848 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:43:57,848 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:43:57,850 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:43:58,218 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:44:00,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:44:00,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:00,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:00,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:05,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:05,332 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:05,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:09,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:44:09,650 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:44:11,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:15,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:15,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:16,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:18,998 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:44:20,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:20,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:22,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:24,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:44:24,654 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:44:27,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:28,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:44:28,862 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:44:28,862 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:44:28,864 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:44:29,253 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:44:30,254 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:44:30,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:32,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:35,429 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:37,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:39,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:44:39,652 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:44:43,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:45,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:45,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:48,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:49,002 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:44:50,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:50,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:44:54,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:44:54,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:44:54,655 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:44:57,724 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:44:57,726 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:44:57,726 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:44:57,728 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:44:58,290 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:44:59,728 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:00,292 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:45:00,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:04,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:05,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:09,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:45:09,655 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:45:09,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:14,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:15,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:19,007 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:45:20,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:20,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:24,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:45:24,669 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:45:25,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:30,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:45:30,542 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:45:30,542 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:45:30,547 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:45:30,687 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:30,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:31,323 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:45:31,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:32,324 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:45:35,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:36,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:39,659 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:45:39,660 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:45:41,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:44,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:44,963 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:46,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:49,011 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:45:49,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:49,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:45:52,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:54,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:45:54,662 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:45:57,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:45:59,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:45:59,352 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:45:59,352 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:45:59,354 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:45:59,358 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:45:59,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:00,359 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:46:03,355 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:04,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:08,356 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:09,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:46:09,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:46:13,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:14,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:18,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:19,016 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:46:19,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:24,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:24,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:46:24,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:46:29,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:29,981 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:34,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:46:34,339 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:46:34,340 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:46:34,341 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:46:34,390 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:46:34,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:35,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:36,392 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:46:39,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:46:39,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:46:40,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:44,980 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:45,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:49,018 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:46:49,982 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:46:51,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:54,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:46:54,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:46:56,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:46:59,996 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:01,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:03,645 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:47:03,646 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:47:03,647 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:47:03,648 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:47:04,428 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:47:04,428 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:47:05,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:05,064 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:07,650 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:09,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:47:09,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:47:12,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:15,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:15,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:17,786 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:19,022 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:47:20,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:20,067 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:23,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:24,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:47:24,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:47:28,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:30,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:33,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:34,202 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:47:34,204 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:47:34,204 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:47:34,205 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:47:34,457 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:47:35,076 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:36,458 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:47:39,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:39,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:47:39,663 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:47:44,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:45,077 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:45,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:49,026 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:47:50,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:47:50,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:50,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:47:54,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:47:54,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:47:55,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:00,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:00,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:00,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:05,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:05,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:48:05,266 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:48:05,266 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:48:05,267 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:48:05,488 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:48:06,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:06,489 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:48:09,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:48:09,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:48:11,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:15,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:16,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:19,030 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:48:20,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:22,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:24,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:48:24,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:48:27,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:30,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:32,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:34,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:48:34,858 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:48:34,859 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:48:34,860 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:48:35,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:35,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:35,524 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:48:36,525 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:48:37,861 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:39,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:48:39,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:48:43,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:45,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:45,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:48,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:49,035 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:48:50,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:50,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:48:54,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:48:54,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:48:54,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:48:59,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:00,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:00,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:04,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:05,291 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:07,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:49:07,913 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:49:07,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:49:07,915 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:49:08,559 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:49:09,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:49:09,664 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:49:10,561 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:49:10,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:15,339 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:15,754 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:19,038 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:49:20,340 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:21,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:24,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:49:24,665 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:49:26,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:30,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:31,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:35,352 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:36,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:39,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:49:39,671 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:49:40,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:49:40,541 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:49:40,542 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:49:40,543 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:49:40,603 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:49:42,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:42,607 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:49:45,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:47,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:49,042 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:49:50,348 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:49:53,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:49:54,667 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:49:54,668 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:49:58,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:00,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:03,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:05,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:05,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:08,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:09,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:50:09,671 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:50:12,002 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:50:12,003 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:50:12,004 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:50:12,006 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:50:12,643 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:50:14,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:14,645 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:50:15,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:19,007 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:19,046 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:50:20,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:24,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:24,671 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:50:24,672 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:50:29,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:30,425 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:30,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:34,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:35,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:35,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:39,678 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:50:39,683 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:50:39,781 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:43,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:50:43,430 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:50:43,431 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:50:43,433 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:50:43,678 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:50:44,680 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:50:45,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:45,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:45,577 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:49,050 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:50:50,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:50:51,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:50:54,675 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:50:54,676 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:50:56,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:00,579 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:00,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:01,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:05,580 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:06,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:09,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:51:09,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:51:11,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:15,586 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:15,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:51:15,814 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:51:15,815 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:51:15,816 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:51:16,717 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:51:16,717 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:51:17,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:19,055 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:51:20,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:20,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:23,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:24,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:51:24,678 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:51:28,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:29,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:29,787 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:33,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:34,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:34,788 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:38,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:39,679 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:51:39,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:51:44,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:51:44,276 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:51:44,276 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:51:44,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:44,278 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:51:44,746 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:51:44,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:46,748 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:51:49,059 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:51:49,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:51:50,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:54,677 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:51:54,678 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:51:55,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:51:59,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:00,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:04,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:05,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:09,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:52:09,680 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:52:11,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:14,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:15,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:52:15,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:52:15,383 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:52:15,385 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:52:15,777 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:52:16,779 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:52:17,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:19,063 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:52:19,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:19,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:23,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:24,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:52:24,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:52:28,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:29,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:29,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:33,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:34,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:34,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:38,772 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:39,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:52:39,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:52:43,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:44,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:45,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:52:45,895 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:52:45,895 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:52:45,897 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:52:46,814 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:52:46,815 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:52:48,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:49,067 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:52:49,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:52:54,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:54,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:52:54,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:52:59,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:52:59,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:04,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:04,819 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:09,691 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:53:09,695 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:53:09,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:14,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:14,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:15,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:53:15,556 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:53:15,557 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:53:15,559 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:53:15,847 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:53:16,849 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:53:19,068 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:53:19,966 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:20,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:24,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:53:24,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:53:25,858 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:30,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:30,041 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:30,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:35,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:35,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:39,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:53:39,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:53:41,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:45,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:46,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:53:46,684 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:53:46,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:53:46,687 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:53:46,892 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:53:47,688 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:48,893 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:53:49,072 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:53:50,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:53:53,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:53:54,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:53:54,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:53:58,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:00,168 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:00,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:03,777 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:05,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:05,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:08,778 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:09,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:54:09,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:54:13,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:15,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:17,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:54:17,757 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:54:17,758 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:54:17,759 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:54:17,928 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:54:18,929 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:54:19,077 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:54:19,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:20,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:24,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:24,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:54:24,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:54:29,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:30,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:34,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:35,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:39,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:54:39,684 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:54:40,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:45,189 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:45,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:49,083 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:54:50,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:50,192 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:54:50,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:54:50,536 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:54:50,536 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:54:50,538 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:54:50,960 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:54:51,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:54:52,962 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:54:54,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:54:54,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:54:56,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:00,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:01,797 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:05,311 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:05,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:06,798 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:09,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:55:09,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:55:12,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:15,327 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:17,770 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:19,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:55:19,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:55:19,088 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:55:19,089 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:55:19,094 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:55:20,002 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:55:20,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:20,506 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:21,003 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:55:23,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:24,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:55:24,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:55:28,765 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:30,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:30,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:33,766 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:35,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:35,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:38,767 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:39,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:55:39,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:55:43,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:45,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:45,526 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:48,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:49,085 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:55:49,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:55:49,843 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:55:49,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:55:49,849 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:55:50,040 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:55:50,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:55:51,041 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:55:53,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:55:54,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:55:54,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:55:59,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:00,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:04,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:05,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:09,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:56:09,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:56:09,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:14,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:15,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:19,091 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:56:20,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:56:20,026 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:56:20,026 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:56:20,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:20,031 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:56:20,075 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:56:20,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:21,076 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:56:24,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:56:24,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:56:25,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:30,532 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:30,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:35,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:35,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:39,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:56:39,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:56:41,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:45,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:46,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:49,100 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:56:49,409 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:56:49,410 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:56:49,411 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:56:49,412 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:56:50,111 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:56:50,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:56:51,112 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:56:52,413 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:54,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:56:54,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:56:57,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:56:59,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:02,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:04,833 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:07,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:09,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:57:09,686 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:57:13,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:14,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:18,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:19,095 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:57:19,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:24,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:57:24,091 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:57:24,092 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:57:24,094 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:24,096 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:57:24,142 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:57:24,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:57:24,686 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:57:25,144 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:57:29,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:29,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:34,779 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:34,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:39,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:57:39,686 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:57:40,773 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:44,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:45,774 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:49,099 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:57:49,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:57:51,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:54,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:57:54,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:57:55,648 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:57:55,650 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:57:55,650 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:57:55,653 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:57:56,176 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:57:56,654 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:57:57,177 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:58:00,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:00,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:01,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:05,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:05,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:06,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:09,685 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:58:09,686 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:58:11,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:15,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:15,061 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:16,760 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:19,103 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:58:20,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:20,062 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:22,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:24,694 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:58:24,694 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:58:27,865 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:28,596 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:58:28,597 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:58:28,597 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:58:28,602 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:58:29,210 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:58:29,210 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:58:30,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:33,603 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:35,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:38,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:39,690 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:58:39,691 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:58:43,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:45,073 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:48,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:49,106 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:58:50,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:58:54,110 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:54,701 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:58:54,701 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:58:59,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:58:59,171 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:58:59,171 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:58:59,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:58:59,176 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:58:59,238 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:59:00,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:01,240 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:59:04,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:05,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:09,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:09,698 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:59:09,699 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:59:14,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:15,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:15,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:19,111 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:59:20,113 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:20,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:24,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:59:24,718 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:59:25,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:30,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:30,649 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 19:59:30,651 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 19:59:30,651 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 19:59:30,656 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 19:59:31,268 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 19:59:31,269 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 19:59:31,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:35,171 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:36,657 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:39,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:59:39,704 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:59:41,867 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:45,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:46,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:49,115 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 19:59:50,173 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 19:59:52,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 19:59:54,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 19:59:54,704 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 19:59:57,843 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:00,183 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:01,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:00:01,360 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:00:01,360 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:00:01,365 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:00:02,301 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:00:03,302 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:00:03,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:05,181 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:08,367 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:09,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:00:09,704 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:00:13,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:15,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:18,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:19,119 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:00:20,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:20,272 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:24,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:24,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:00:24,704 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:00:29,771 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:30,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:34,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:00:34,278 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:00:34,279 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:00:34,284 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:00:34,335 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:00:35,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:35,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:35,336 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:00:39,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:00:39,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:00:40,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:45,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:45,852 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:49,123 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:00:50,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:00:51,126 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:00:54,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:00:54,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:00:56,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:00,299 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:01,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:01,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:01:01,970 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:01:01,970 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:01:01,972 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:01:02,364 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:01:03,365 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:01:05,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:06,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:09,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:01:09,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:01:12,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:15,376 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:17,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:19,127 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:01:20,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:23,130 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:24,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:01:24,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:01:28,783 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:30,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:32,651 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:01:32,652 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:01:32,653 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:01:32,655 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:01:33,406 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:01:33,407 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:01:34,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:35,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:39,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:39,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:01:39,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:01:44,868 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:45,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:49,131 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:01:50,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:01:50,391 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:01:54,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:01:54,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:01:55,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:00,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:00,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:04,828 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:02:04,829 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:02:04,830 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:02:04,831 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:02:05,402 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:05,439 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:02:05,440 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:02:06,832 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:09,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:02:09,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:02:12,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:15,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:17,806 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:19,136 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:02:20,404 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:23,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:24,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:02:24,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:02:28,872 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:30,419 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:33,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:35,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:36,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:02:36,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:02:36,395 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:02:36,396 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:02:36,471 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:02:37,472 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:02:39,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:39,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:02:39,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:02:44,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:45,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:45,542 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:49,140 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:02:50,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:02:50,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:50,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:02:54,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:02:54,705 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:02:55,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:00,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:00,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:05,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:05,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:05,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:03:05,915 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:03:05,916 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:03:05,917 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:03:06,503 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:03:07,505 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:03:09,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:03:09,706 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:03:10,876 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:15,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:15,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:19,144 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:03:20,549 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:21,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:24,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:03:24,706 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:03:26,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:30,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:31,823 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:35,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:36,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:39,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:03:39,600 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:03:39,600 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:03:39,602 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:03:40,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:03:40,054 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:03:40,549 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:03:41,550 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:03:42,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:45,565 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:47,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:49,148 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:03:50,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:03:53,150 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:03:54,749 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:03:54,749 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:03:58,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:00,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:00,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:03,877 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:05,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:05,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:08,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:09,485 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:04:09,486 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:04:09,487 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:04:09,488 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:04:09,582 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:04:09,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:04:09,891 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:04:11,584 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:04:14,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:15,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:15,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:19,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:19,152 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:04:20,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:20,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:24,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:24,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:04:24,890 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:04:29,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:29,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:30,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:34,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:35,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:39,900 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:04:39,900 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:04:40,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:42,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:04:42,255 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:04:42,255 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:04:42,257 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:04:42,622 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:04:43,624 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:04:44,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:45,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:49,156 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:04:49,903 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:51,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:54,893 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:04:54,894 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:04:57,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:04:59,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:04:59,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:02,033 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:04,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:04,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:07,034 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:09,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:05:09,910 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:05:13,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:14,922 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:15,680 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:05:15,681 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:05:15,681 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:05:15,683 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:05:16,656 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:05:18,657 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:05:18,684 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:19,160 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:05:20,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:24,163 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:24,897 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:05:24,897 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:05:29,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:30,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:30,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:34,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:35,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:35,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:39,910 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:05:39,911 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:05:39,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:44,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:45,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:45,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:45,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:05:45,387 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:05:45,387 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:05:45,389 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:05:45,687 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:05:48,690 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:05:49,164 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:05:50,103 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:50,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:05:50,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:05:54,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:05:54,903 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:05:56,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:00,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:00,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:01,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:05,108 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:05,109 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:06,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:09,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:06:09,919 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:06:11,987 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:15,119 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:15,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:15,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:06:15,296 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:06:15,297 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:06:15,299 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:06:15,719 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:06:16,720 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:06:17,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:19,168 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:06:20,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:20,121 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:23,170 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:24,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:06:24,906 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:06:29,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:30,122 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:30,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:34,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:35,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:35,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:39,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:39,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:06:39,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:06:44,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:06:44,314 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:06:44,314 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:06:44,314 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:44,316 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:06:44,752 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:06:45,133 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:46,755 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:06:49,170 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:06:50,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:06:50,172 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:06:54,909 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:06:54,910 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:06:56,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:00,136 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:01,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:05,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:06,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:09,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:07:09,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:07:11,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:13,438 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:07:13,440 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:07:13,441 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:07:13,442 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:07:13,785 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:07:14,786 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:07:15,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:17,444 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:19,174 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:07:20,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:23,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:24,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:07:24,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:07:28,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:30,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:33,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:35,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:38,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:39,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:07:39,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:07:44,006 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:45,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:47,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:07:47,472 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:07:47,473 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:07:47,477 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:07:47,824 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:07:48,825 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:07:49,179 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:07:49,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:50,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:07:54,182 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:07:54,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:07:54,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:07:59,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:00,279 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:04,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:05,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:09,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:08:09,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:08:10,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:15,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:15,318 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:19,186 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:08:20,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:20,319 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:21,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:08:21,395 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:08:21,396 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:08:21,398 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:08:21,862 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:08:22,864 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:08:24,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:08:24,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:08:26,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:30,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:31,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:35,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:35,322 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:36,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:39,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:08:39,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:08:41,991 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:45,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:45,336 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:46,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:49,190 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:08:50,329 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:08:51,211 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:08:51,211 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:08:51,212 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:08:51,213 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:08:51,893 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:08:52,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:08:52,894 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:08:54,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:08:54,913 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:08:58,078 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:00,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:03,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:05,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:05,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:08,079 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:09,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:09:09,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:09:14,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:15,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:19,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:19,187 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:09:20,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:21,294 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:09:21,296 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:09:21,296 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:09:21,299 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:09:21,924 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:09:22,925 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:09:24,300 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:24,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:09:24,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:09:30,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:30,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:30,721 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:35,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:35,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:35,722 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:39,913 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:09:39,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:09:41,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:45,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:46,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:49,199 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:09:50,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:09:50,656 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:09:50,656 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:09:50,657 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:09:50,805 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:50,958 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:09:51,070 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:09:51,658 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:09:52,960 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:09:54,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:09:54,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:09:57,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:00,071 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:02,035 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:05,072 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:07,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:09,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:10:09,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:10:13,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:15,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:18,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:18,445 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:10:18,447 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:10:18,447 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:10:18,449 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:10:19,002 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:10:19,196 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:10:20,080 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:21,004 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:10:23,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:24,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:10:24,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:10:29,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:30,082 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:34,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:35,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:39,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:39,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:10:39,914 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:10:45,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:45,084 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:49,212 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:10:50,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:10:50,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:54,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:10:54,198 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:10:54,199 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:10:54,200 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:10:54,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:10:54,915 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:10:55,041 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:10:56,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:10:57,043 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:11:00,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:01,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:05,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:06,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:09,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:11:09,915 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:11:12,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:15,097 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:17,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:19,205 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:11:20,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:22,207 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:24,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:11:24,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:11:27,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:11:27,340 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:11:27,340 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:11:27,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:27,344 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:11:28,075 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:11:29,076 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:11:30,302 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:32,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:35,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:35,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:37,347 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:39,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:11:39,919 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:11:43,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:45,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:45,305 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:48,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:49,215 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:11:50,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:11:53,219 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:11:54,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:11:54,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:11:55,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:11:55,113 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:11:55,113 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:11:55,115 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:11:56,105 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:11:57,106 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:11:59,116 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:00,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:00,523 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:04,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:05,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:05,524 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:09,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:09,918 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:12:09,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:12:15,055 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:15,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:15,605 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:19,212 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:12:20,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:20,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:24,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:12:24,921 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:12:25,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:26,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:12:26,385 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:12:26,385 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:12:26,387 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:12:27,140 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:12:29,142 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:12:30,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:30,707 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:31,388 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:35,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:35,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:36,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:39,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:12:39,920 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:12:42,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:45,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:45,711 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:47,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:49,217 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:12:50,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:12:52,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:54,920 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:12:54,922 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:12:56,729 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:12:56,730 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:12:56,731 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:12:56,732 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:12:57,173 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:12:57,732 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:12:59,174 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:13:00,742 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:02,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:05,743 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:05,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:07,734 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:10,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:13:10,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:13:13,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:15,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:15,745 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:18,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:19,221 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:13:20,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:20,746 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:23,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:24,951 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:13:24,951 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:13:28,142 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:13:28,144 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:13:28,145 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:13:28,147 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:13:28,206 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:13:29,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:30,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:31,209 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:13:34,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:35,753 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:39,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:39,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:13:39,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:13:45,068 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:45,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:49,225 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:13:50,227 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:13:50,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:13:54,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:13:54,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:13:56,088 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:00,759 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:01,090 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:01,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:14:01,372 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:14:01,373 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:14:01,374 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:14:02,259 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:14:03,260 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:14:05,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:05,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:06,375 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:09,947 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:14:09,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:14:12,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:15,791 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:17,048 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:19,229 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:14:20,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:20,792 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:22,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:24,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:14:24,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:14:28,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:30,800 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:33,059 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:33,944 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:14:33,946 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:14:33,946 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:14:33,948 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:14:34,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:14:35,298 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:14:35,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:38,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:39,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:14:39,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:14:44,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:45,803 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:49,075 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:49,233 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:14:50,804 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:14:54,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:14:54,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:14:54,948 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:15:00,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:00,814 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:04,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:15:04,746 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:15:04,746 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:15:04,747 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:15:05,330 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:15:05,748 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:06,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:06,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:07,331 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:15:09,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:15:09,949 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:15:11,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:15,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:16,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:19,237 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:15:20,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:20,166 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:21,239 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:24,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:15:24,949 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:15:27,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:30,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:32,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:35,175 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:37,104 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:38,083 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:15:38,084 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:15:38,084 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:15:38,086 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:15:38,370 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:15:39,372 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:15:39,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:15:39,949 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:15:43,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:45,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:48,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:49,241 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:15:50,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:15:53,244 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:15:54,948 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:15:54,949 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:15:59,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:00,238 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:04,066 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:05,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:09,069 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:09,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:16:09,684 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:16:09,685 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:16:09,686 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:16:10,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:16:10,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:16:10,395 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:16:11,396 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:16:14,240 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:15,247 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:19,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:19,245 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:16:20,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:24,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:25,098 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:16:25,098 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:16:30,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:30,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:35,199 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:35,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:40,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:16:40,103 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:16:40,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:41,488 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:16:41,490 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:16:41,491 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:16:41,493 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:16:42,430 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:16:43,431 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:16:45,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:45,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:49,248 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:16:50,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:16:51,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:16:55,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:16:55,102 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:16:56,269 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:00,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:01,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:05,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:06,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:07,448 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:17:11,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:14,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:14,427 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:15,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:17:15,331 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:17:17,439 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:19,252 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:17:22,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:22,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:23,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:28,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:31,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:32,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:32,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:17:32,345 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:17:33,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:38,500 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:40,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:40,606 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:43,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:48,501 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:48,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:49,256 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:17:49,782 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:17:49,782 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:17:53,925 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:17:57,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:57,950 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:17:58,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:03,616 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:18:03,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:18:03,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:05,664 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:18:08,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:09,543 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:13,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:14,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:14,544 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:18,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:18:18,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:18:19,258 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:18:19,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:24,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:24,545 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:29,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:29,546 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:33,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:18:33,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:18:34,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:39,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:39,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:44,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:44,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:48,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:18:48,617 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:18:49,262 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:18:50,263 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:18:54,550 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:54,551 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:18:55,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:00,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:05,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:10,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:15,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:19,265 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:19:19,830 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:19,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:19:19,841 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:19:21,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:24,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:26,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:29,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:31,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:34,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:19:34,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:19:36,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:39,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:41,978 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:44,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:46,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:49,269 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:19:49,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:19:49,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:19:52,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:54,841 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:19:57,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:19:59,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:02,972 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:04,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:20:04,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:20:08,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:09,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:13,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:14,844 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:18,971 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:19,273 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:20:19,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:20:19,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:20:24,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:24,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:29,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:29,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:34,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:34,836 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:20:34,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:20:39,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:39,993 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:44,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:44,994 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:49,277 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:20:49,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:20:49,838 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:20:50,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:54,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:20:55,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:20:59,851 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:00,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:04,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:21:04,837 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:21:06,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:11,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:16,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:19,281 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:21:22,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:27,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:32,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:37,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:42,287 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:45,780 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:45,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:21:45,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:21:47,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:49,286 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:21:49,907 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:21:50,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:50,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:53,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:21:53,910 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:21:55,821 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:55,822 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:21:58,720 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:00,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:22:00,786 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:22:03,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:05,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:05,824 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:08,892 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:10,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:10,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:13,896 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:15,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:22:15,803 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:22:18,860 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:22:18,862 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:22:18,863 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:22:18,864 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:22:18,940 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:22:19,289 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:22:19,295 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:20,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:20,834 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:21,943 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:22:24,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:25,835 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:29,292 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:30,789 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:22:30,790 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:22:34,911 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:35,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:39,912 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:40,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:40,838 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:44,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:45,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:22:45,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:22:49,292 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:22:50,297 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:22:50,845 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:51,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:22:51,022 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:22:51,023 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:22:51,024 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:22:51,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:22:53,976 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:22:55,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:22:56,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:00,793 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:23:00,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:23:01,927 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:05,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:05,848 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:06,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:10,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:10,849 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:11,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:15,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:23:15,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:23:16,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:19,301 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:23:20,859 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:22,303 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:22,307 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:23:22,308 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:23:22,309 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:23:22,311 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:23:23,008 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:23:24,009 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:23:25,857 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:27,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:30,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:23:30,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:23:32,380 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:35,916 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:37,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:40,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:40,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:42,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:45,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:23:45,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:23:47,934 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:49,306 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:23:50,929 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:53,312 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:23:53,846 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:23:53,848 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:23:53,848 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:23:53,853 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:23:54,041 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:23:56,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:23:56,043 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:23:58,855 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:00,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:24:00,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:24:03,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:06,003 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:08,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:11,004 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:13,942 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:15,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:24:15,794 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:24:18,958 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:19,309 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:24:21,015 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:24,224 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:24:24,225 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:24:24,226 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:24:24,226 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:24,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:24:25,080 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:24:26,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:26,081 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:24:29,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:30,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:24:30,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:24:34,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:36,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:39,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:41,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:44,924 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:45,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:24:45,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:24:49,315 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:24:50,320 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:51,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:24:53,842 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:24:53,844 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:24:53,844 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:24:53,846 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:24:54,116 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:24:55,847 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:24:56,118 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:24:56,255 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:00,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:25:00,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:25:00,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:05,904 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:06,257 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:10,905 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:11,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:15,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:25:15,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:25:15,959 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:19,312 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:25:21,264 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:21,313 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:23,401 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:25:23,403 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:25:23,404 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:25:23,405 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:25:24,148 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:25:24,148 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:25:26,265 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:26,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:30,794 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:25:30,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:25:31,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:36,266 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:36,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:41,267 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:41,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:45,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:25:45,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:25:46,949 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:49,320 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:25:51,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:52,321 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:25:52,708 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:25:52,710 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:25:52,710 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:25:52,712 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:25:53,176 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:25:54,177 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:25:56,275 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:25:57,713 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:00,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:26:00,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:26:02,914 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:06,276 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:07,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:11,293 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:12,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:15,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:26:15,795 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:26:17,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:19,327 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:26:21,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:23,331 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:25,054 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:26:25,055 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:26:25,055 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:26:25,057 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:26:25,215 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:26:26,217 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:26:26,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:29,058 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:30,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:26:30,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:26:34,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:36,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:39,960 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:41,467 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:44,961 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:45,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:26:45,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:26:49,327 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:26:50,330 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:51,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:55,338 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:26:56,472 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:26:56,954 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:26:56,955 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:26:56,956 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:26:56,959 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:26:57,255 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:26:58,256 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:27:00,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:27:00,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:27:00,930 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:05,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:05,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:05,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:10,931 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:10,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:15,795 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:27:15,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:27:16,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:19,331 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:27:20,986 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:22,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:25,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:27,343 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:28,595 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:27:28,597 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:27:28,597 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:27:28,602 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:27:29,297 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:27:30,298 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:27:30,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:27:30,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:27:32,939 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:36,017 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:37,940 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:41,018 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:42,941 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:45,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:27:45,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:27:48,915 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:49,335 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:27:51,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:54,341 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:27:56,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:27:59,270 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:27:59,271 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:27:59,271 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:27:59,276 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:27:59,331 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:28:00,277 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:00,332 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:28:00,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:28:00,796 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:28:05,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:06,029 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:10,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:11,030 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:15,796 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:28:15,797 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:28:16,880 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:19,339 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:28:21,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:22,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:26,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:27,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:30,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:28:30,807 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:28:32,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:34,604 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:28:34,605 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:28:34,605 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:28:34,610 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:28:35,362 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:28:36,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:36,363 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:28:38,611 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:41,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:43,612 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:45,799 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:28:45,800 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:28:48,946 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:49,344 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:28:51,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:54,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:28:56,040 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:28:59,349 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:00,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:29:00,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:29:02,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:29:02,822 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:29:02,822 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:29:02,824 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:29:03,387 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:29:04,388 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:29:04,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:06,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:06,248 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:09,826 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:11,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:11,249 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:14,827 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:15,802 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:29:15,803 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:29:19,348 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:29:20,350 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:21,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:21,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:25,351 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:26,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:26,251 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:30,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:30,811 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:29:30,813 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:29:32,656 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:29:32,658 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:29:32,658 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:29:32,660 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:29:33,420 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:29:34,422 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:29:35,661 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:36,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:40,662 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:41,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:45,663 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:45,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:29:45,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:29:49,351 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:29:51,353 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:51,383 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:51,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:56,354 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:29:56,384 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:29:56,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:00,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:30:00,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:30:01,923 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:05,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:30:05,394 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:30:05,395 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:30:05,396 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:30:05,450 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:30:06,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:06,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:06,451 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:30:07,397 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:11,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:12,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:15,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:30:15,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:30:17,967 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:19,355 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:30:21,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:21,539 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:23,357 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:26,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:26,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:28,358 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:30,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:30:30,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:30:33,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:36,547 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:38,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:39,554 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:30:39,556 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:30:39,557 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:30:39,559 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:30:40,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:30:40,492 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:30:41,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:41,556 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:44,560 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:45,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:30:45,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:30:49,358 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:30:50,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:51,557 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:51,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:55,361 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:30:56,558 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:30:56,559 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:00,362 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:00,807 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:31:00,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:31:05,926 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:06,568 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:08,917 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:31:08,918 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:31:08,918 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:31:08,920 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:31:09,523 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:31:10,524 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:31:11,569 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:11,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:15,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:31:15,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:31:17,887 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:19,362 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:31:21,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:21,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:23,365 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:26,784 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:26,785 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:28,366 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:30,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:31:30,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:31:34,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:36,790 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:38,562 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:31:38,564 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:31:38,564 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:31:38,565 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:31:39,556 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:31:39,566 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:40,557 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:31:41,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:41,988 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:44,567 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:45,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:31:45,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:31:49,366 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:31:50,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:50,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:50,989 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:55,369 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:31:55,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:31:55,990 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:00,370 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:00,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:32:00,808 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:32:05,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:06,005 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:08,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:32:08,639 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:32:08,639 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:32:08,641 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:32:09,589 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:32:10,590 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:32:11,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:11,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:11,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:15,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:32:15,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:32:16,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:19,370 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:32:21,053 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:22,373 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:26,074 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:27,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:30,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:32:30,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:32:32,879 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:36,081 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:37,891 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:41,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:41,131 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:32:41,134 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:32:41,134 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:32:41,136 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:32:41,618 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:32:42,619 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:32:43,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:45,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:32:45,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:32:48,889 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:49,375 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:32:51,087 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:54,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:32:56,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:32:59,378 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:00,808 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:33:00,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:33:04,921 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:06,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:09,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:11,161 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:12,105 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:33:12,107 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:33:12,108 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:33:12,110 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:33:12,652 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:33:14,654 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:33:15,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:15,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:33:15,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:33:19,379 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:33:20,381 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:21,529 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:25,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:26,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:26,530 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:30,382 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:30,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:33:30,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:33:35,874 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:36,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:40,878 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:41,535 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:42,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:33:42,441 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:33:42,441 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:33:42,443 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:33:42,684 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:33:44,686 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:33:45,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:33:45,809 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:33:45,943 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:49,383 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:33:51,385 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:51,589 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:33:56,386 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:33:56,590 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:00,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:34:00,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:34:01,928 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:06,601 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:06,935 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:10,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:34:10,028 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:34:10,028 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:34:10,030 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:34:10,706 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:34:10,706 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:34:11,597 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:12,031 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:15,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:34:15,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:34:17,936 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:19,387 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:34:21,598 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:23,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:26,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:28,390 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:30,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:34:30,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:34:33,964 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:36,609 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:38,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:41,617 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:42,816 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:34:42,817 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:34:42,817 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:34:42,819 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:34:43,741 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:34:44,742 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:34:44,820 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:45,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:34:45,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:34:49,391 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:34:50,393 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:51,618 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:34:55,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:34:56,619 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:00,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:00,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:35:00,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:35:05,890 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:06,620 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:10,894 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:11,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:15,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:35:15,227 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:35:15,228 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:35:15,230 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:35:15,769 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:35:15,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:35:15,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:35:15,973 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:16,770 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:35:19,396 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:35:21,398 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:21,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:26,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:26,628 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:30,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:35:30,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:35:31,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:36,630 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:36,970 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:41,637 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:41,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:45,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:35:45,810 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:35:46,120 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:35:46,121 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:35:46,122 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:35:46,123 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:35:46,798 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:35:46,799 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:35:47,123 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:49,396 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:35:51,640 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:52,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:35:56,641 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:35:57,399 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:00,810 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:36:00,811 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:36:02,932 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:06,642 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:07,933 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:11,653 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:12,937 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:15,813 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:36:15,813 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:36:16,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:36:16,707 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:36:16,707 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:36:16,709 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:36:16,831 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:36:18,709 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:18,832 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:36:19,400 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:36:20,853 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:24,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:25,854 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:29,403 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:30,898 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:30,901 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:36:30,902 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:36:34,968 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:36,655 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:39,969 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:41,664 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:44,979 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:45,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:36:45,899 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:36:46,676 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:36:46,678 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:36:46,679 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:36:46,680 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:36:46,858 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:36:48,860 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:36:49,404 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:36:50,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:51,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:36:55,407 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:36:56,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:00,408 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:00,899 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:37:00,900 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:37:06,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:06,095 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:11,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:11,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:15,902 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:37:15,902 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:37:16,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:17,744 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:37:17,746 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:37:17,746 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:37:17,748 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:37:17,886 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:37:18,888 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:37:19,408 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:37:21,157 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:22,411 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:26,158 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:27,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:31,101 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:37:31,101 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:37:33,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:36,160 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:38,185 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:41,169 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:43,194 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:46,008 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:37:46,009 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:37:47,124 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:37:47,126 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:37:47,127 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:37:47,128 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:37:47,912 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:37:48,913 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:37:49,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:49,413 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:37:51,284 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:54,415 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:37:56,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:37:59,416 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:01,009 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:38:01,010 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:38:05,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:06,345 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:10,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:11,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:15,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:16,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:38:16,019 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:38:19,419 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:38:20,052 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:38:20,054 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:38:20,054 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:38:20,055 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:38:20,944 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:38:20,945 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:38:21,056 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:21,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:26,057 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:26,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:31,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:38:31,013 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:38:31,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:36,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:36,492 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:41,178 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:41,493 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:46,016 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:38:46,018 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:38:47,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:48,608 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:38:48,609 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:38:48,609 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:38:48,611 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:38:48,973 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:38:49,420 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:38:50,975 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:38:51,504 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:52,422 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:38:56,505 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:38:57,423 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:01,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:39:01,013 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:39:03,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:06,513 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:08,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:11,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:13,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:16,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:39:16,014 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:39:19,177 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:19,427 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:39:20,389 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:39:20,391 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:39:20,391 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:39:20,392 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:39:21,005 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:39:21,006 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:39:21,537 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:24,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:26,538 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:29,394 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:31,014 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:39:31,015 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:39:35,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:36,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:40,128 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:41,540 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:45,129 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:46,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:39:46,025 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:39:49,439 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:39:50,440 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:51,548 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:39:52,304 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:39:52,306 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:39:52,306 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:39:52,308 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:39:53,053 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:39:53,054 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:39:56,309 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:39:56,717 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:01,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:40:01,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:40:02,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:06,718 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:07,148 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:11,719 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:12,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:16,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:40:16,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:40:18,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:19,441 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:40:21,733 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:23,342 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:40:23,343 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:40:23,344 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:40:23,344 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:23,345 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:40:24,085 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:40:25,086 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:40:26,761 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:28,346 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:31,019 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:40:31,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:40:34,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:36,762 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:39,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:41,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:44,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:46,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:40:46,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:40:49,115 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:49,441 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:40:51,775 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:54,449 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:40:56,776 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:40:56,975 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:40:56,977 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:40:56,977 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:40:56,982 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:40:57,119 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:40:59,120 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:40:59,984 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:01,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:41:01,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:41:05,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:06,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:06,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:10,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:11,065 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:15,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:16,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:41:16,020 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:41:19,445 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:41:20,447 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:21,145 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:21,146 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:25,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:26,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:29,585 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:41:29,587 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:41:29,588 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:41:29,593 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:41:30,155 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:41:30,594 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:31,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:41:31,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:41:31,156 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:41:36,106 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:36,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:36,153 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:41,107 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:41,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:41,154 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:46,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:41:46,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:41:46,159 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:49,448 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:41:51,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:51,156 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:51,450 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:56,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:41:56,455 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:41:59,434 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:41:59,436 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:41:59,436 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:41:59,437 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:42:00,185 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:42:01,021 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:42:01,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:42:01,186 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:42:02,137 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:06,164 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:07,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:11,165 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:12,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:16,020 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:42:16,021 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:42:18,138 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:19,452 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:42:21,167 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:23,454 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:26,174 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:28,465 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:31,025 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:42:31,025 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:42:32,085 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:42:32,086 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:42:32,086 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:42:32,091 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:42:32,220 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:42:33,221 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:42:34,091 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:36,234 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:39,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:41,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:44,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:46,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:42:46,023 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:42:49,187 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:49,456 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:42:51,236 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:54,459 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:42:56,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:42:59,464 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:01,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:43:01,033 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:43:04,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:43:04,517 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:43:04,517 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:43:04,517 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:04,520 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:43:05,252 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:43:05,252 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:43:06,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:09,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:11,243 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:14,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:16,032 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:43:16,033 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:43:19,460 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:43:20,462 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:21,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:25,463 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:26,245 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:30,469 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:31,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:43:31,049 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:43:34,193 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:43:34,194 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:43:34,195 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:43:34,196 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:43:34,282 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:43:36,258 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:36,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:36,283 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:43:41,259 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:41,260 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:46,036 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:43:46,037 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:43:47,162 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:49,463 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:43:51,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:52,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:43:56,261 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:43:57,466 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:01,044 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:44:01,045 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:44:03,209 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:06,262 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:44:06,263 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:44:06,264 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:44:06,267 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:44:06,313 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:44:06,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:08,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:08,314 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:44:11,681 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:13,268 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:16,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:44:16,039 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:44:19,135 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:19,468 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:44:21,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:24,470 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:26,683 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:29,471 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:31,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:44:31,039 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:44:35,179 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:36,705 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:37,809 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:44:37,811 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:44:37,812 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:44:37,816 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:44:38,350 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:44:40,351 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:44:40,817 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:41,702 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:45,818 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:46,038 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:44:46,039 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:44:49,471 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:44:51,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:51,703 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:44:56,474 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:44:56,704 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:01,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:45:01,039 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:45:02,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:06,710 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:07,210 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:08,418 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:45:08,420 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:45:08,420 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:45:08,425 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:45:09,375 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:45:10,375 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:45:11,837 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:12,426 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:16,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:45:16,039 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:45:18,191 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:19,475 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:45:21,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:21,839 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:23,478 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:26,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:26,840 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:28,479 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:31,042 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:45:31,042 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:45:34,134 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:35,578 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:45:35,579 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:45:35,580 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:45:35,581 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:45:36,406 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:45:36,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:36,998 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:38,408 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:45:39,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:42,000 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:44,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:46,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:45:46,040 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:45:49,480 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:45:50,482 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:52,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:45:55,483 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:45:57,001 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:00,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:01,051 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:46:01,052 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:46:06,155 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:06,424 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:46:06,426 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:46:06,427 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:46:06,429 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:46:06,441 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:46:07,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:07,010 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:08,443 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:46:11,430 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:12,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:12,011 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:16,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:46:16,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:46:17,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:19,484 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:46:22,012 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:22,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:22,486 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:27,013 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:27,487 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:31,045 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:46:31,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:46:33,176 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:37,024 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:38,184 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:38,374 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:46:38,376 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:46:38,376 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:46:38,378 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:46:38,486 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:46:40,487 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:46:42,026 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:43,379 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:46,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:46:46,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:46:49,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:49,487 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:46:52,027 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:54,490 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:46:57,028 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:46:59,491 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:47:01,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:47:05,222 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:07,037 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:09,237 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:47:09,239 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:47:09,239 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:47:09,241 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:47:09,520 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:47:10,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:10,521 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:47:12,039 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:15,242 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:16,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:47:16,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:47:19,492 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:47:20,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:21,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:21,204 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:25,494 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:26,205 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:26,206 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:30,495 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:31,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:47:31,046 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:47:36,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:36,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:39,231 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:47:39,232 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:47:39,232 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:47:39,234 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:47:39,547 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:47:40,548 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:47:41,213 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:41,235 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:46,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:47:46,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:47:47,217 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:49,496 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:47:51,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:52,498 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:47:56,241 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:47:57,499 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:48:01,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:48:03,118 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:06,254 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:08,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:11,250 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:12,428 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:48:12,429 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:48:12,430 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:48:12,432 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:48:12,579 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:48:13,433 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:14,580 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:48:16,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:48:16,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:48:19,139 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:19,500 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:48:21,252 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:24,502 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:26,278 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:29,503 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:31,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:48:31,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:48:35,197 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:36,285 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:40,203 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:41,286 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:42,229 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:48:42,231 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:48:42,231 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:48:42,232 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:48:42,611 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:48:44,612 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:48:45,233 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:46,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:48:46,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:48:49,504 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:48:50,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:51,377 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:55,507 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:48:56,412 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:48:56,581 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:00,508 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:01,046 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:49:01,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:49:06,117 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:06,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:06,583 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:11,125 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:11,599 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:15,317 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:49:15,319 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:49:15,319 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:49:15,320 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:49:15,637 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:49:16,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:49:16,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:49:16,212 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:16,638 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:49:19,508 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:49:21,510 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:21,755 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:21,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:26,511 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:26,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:26,756 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:31,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:49:31,047 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:49:32,143 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:36,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:36,758 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:37,144 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:41,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:42,149 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:44,022 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:49:44,023 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:49:44,024 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:49:44,025 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:49:44,662 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:49:46,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:49:46,048 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:49:46,663 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:49:47,151 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:49,512 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:49:51,763 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:52,514 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:49:56,825 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:57,049 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:49:57,515 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:01,047 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:50:01,048 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:50:03,214 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:06,050 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:08,215 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:11,060 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:13,223 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:15,665 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:50:15,666 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:50:15,667 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:50:15,668 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:50:15,700 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:50:16,092 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:16,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:16,096 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:50:16,097 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:50:18,703 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:50:19,186 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:19,516 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:50:21,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:21,099 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:24,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:26,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:26,100 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:29,519 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:31,093 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:50:31,093 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:50:35,216 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:36,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:36,102 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:40,220 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:41,111 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:45,225 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:45,871 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 20:50:45,873 DEBUG SenderThread:2690175 [sender.py:send():379] send: history +2025-01-01 20:50:45,874 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: summary_record +2025-01-01 20:50:45,875 INFO SenderThread:2690175 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 20:50:46,280 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:46,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:46,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:50:46,282 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status +2025-01-01 20:50:46,731 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/wandb-summary.json +2025-01-01 20:50:48,733 INFO Thread-12 :2690175 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/files/output.log +2025-01-01 20:50:49,519 DEBUG SenderThread:2690175 [sender.py:send():379] send: stats +2025-01-01 20:50:50,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:52,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:52,282 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:55,521 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:50:57,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:50:57,283 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 20:51:00,522 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 20:51:01,281 DEBUG HandlerThread:2690175 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 20:51:01,282 DEBUG SenderThread:2690175 [sender.py:send_request():406] send_request: stop_status