diff --git "a/wandb/run-20250102_021927-pw8rud5e/logs/debug-internal.log" "b/wandb/run-20250102_021927-pw8rud5e/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20250102_021927-pw8rud5e/logs/debug-internal.log" @@ -0,0 +1,13235 @@ +2025-01-02 02:19:27,838 INFO StreamThr :2087140 [internal.py:wandb_internal():85] W&B internal server running at pid: 2087140, started at: 2025-01-02 02:19:27.836981 +2025-01-02 02:19:27,840 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status +2025-01-02 02:19:27,848 INFO WriterThread:2087140 [datastore.py:open_for_write():87] open: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/run-pw8rud5e.wandb +2025-01-02 02:19:27,849 DEBUG SenderThread:2087140 [sender.py:send():379] send: header +2025-01-02 02:19:27,880 DEBUG SenderThread:2087140 [sender.py:send():379] send: run +2025-01-02 02:19:28,115 INFO SenderThread:2087140 [dir_watcher.py:__init__():211] watching files in: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files +2025-01-02 02:19:28,115 INFO SenderThread:2087140 [sender.py:_start_run_threads():1188] run started: pw8rud5e with start time 1735784367.839593 +2025-01-02 02:19:28,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: check_version +2025-01-02 02:19:28,137 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: check_version +2025-01-02 02:19:28,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: run_start +2025-01-02 02:19:28,237 DEBUG HandlerThread:2087140 [system_info.py:__init__():26] System info init +2025-01-02 02:19:28,237 DEBUG HandlerThread:2087140 [system_info.py:__init__():41] System info init done +2025-01-02 02:19:28,237 INFO HandlerThread:2087140 [system_monitor.py:start():194] Starting system monitor +2025-01-02 02:19:28,237 INFO SystemMonitor:2087140 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-02 02:19:28,237 INFO HandlerThread:2087140 [system_monitor.py:probe():214] Collecting system info +2025-01-02 02:19:28,238 INFO SystemMonitor:2087140 [interfaces.py:start():188] Started cpu monitoring +2025-01-02 02:19:28,238 INFO SystemMonitor:2087140 [interfaces.py:start():188] Started disk monitoring +2025-01-02 02:19:28,239 INFO SystemMonitor:2087140 [interfaces.py:start():188] Started gpu monitoring +2025-01-02 02:19:28,240 INFO SystemMonitor:2087140 [interfaces.py:start():188] Started memory monitoring +2025-01-02 02:19:28,242 INFO SystemMonitor:2087140 [interfaces.py:start():188] Started network monitoring +2025-01-02 02:19:28,293 DEBUG HandlerThread:2087140 [system_info.py:probe():152] Probing system +2025-01-02 02:19:28,296 DEBUG HandlerThread:2087140 [system_info.py:_probe_git():137] Probing git +2025-01-02 02:19:28,317 DEBUG HandlerThread:2087140 [system_info.py:_probe_git():145] Probing git done +2025-01-02 02:19:28,317 DEBUG HandlerThread:2087140 [system_info.py:probe():200] Probing system done +2025-01-02 02:19:28,317 DEBUG HandlerThread:2087140 [system_monitor.py:probe():223] {'os': 'Linux-6.8.0-48-generic-x86_64-with-glibc2.39', 'python': '3.10.0', 'heartbeatAt': '2025-01-02T02:19:28.293281', 'startedAt': '2025-01-02T02:19:27.822029', 'docker': None, 'cuda': None, 'args': ('--model_family', 'llama', '--apply_instruct_masks', '--token_scaled_loss', '--seq_parallel_size', '8', '--report_to', 'wandb', '--do_train', '--model_name_or_path', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--config_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--tokenizer_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--run_name', '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--output_dir', '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--config_overrides_json', '', '--gradient_accumulation_steps', '2', '--per_device_train_batch_size', '1', '--bf16', '--learning_rate', '1e-6', '--min_lr_ratio', '0.1', '--lr_scheduler_type', 'cosine', '--max_grad_norm', '1.0', '--adam_beta1', '0.9', '--adam_beta2', '0.95', '--weight_decay', '0.1', '--warmup_ratio', '0.05', '--optim', 'adamw_torch', '--logging_steps', '1', '--log_level', 'info', '--save_steps', '200', '--dataloader_num_workers', '1', '--disable_tqdm', 'true', '--use_fast_tokenizer', 'false', '--remove_unused_columns', 'false', '--ddp_find_unused_parameters', 'false', '--fsdp', 'auto_wrap offload', '--gradient_checkpointing', '--tokenized_mds_train', '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete', '--cuda_empty_cache', '--num_train_epochs', '1'), 'state': 'running', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'codePathLocal': 'finetune.py', 'codePath': 'prolong-final/finetune.py', 'git': {'remote': 'https://github.com/chtmp223/BookGen-dev.git', 'commit': '0e796521430a0f767be7c4dadba5c2fcaee1f909'}, 'email': 'ctpham@umass.edu', 'root': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev', 'host': 'gpu020', 'username': 'ctpham_umass_edu', 'executable': '/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10', 'cpu_count': 112, 'cpu_count_logical': 112, 'cpu_freq': {'current': 958.6112589285714, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 784.548, 'min': 800.0, 'max': 3800.0}, {'current': 761.056, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.579, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.219, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1500.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 786.524, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1994.764, 'min': 800.0, 'max': 3800.0}, {'current': 784.651, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1717.12, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1300.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2005.181, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.387, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 782.915, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.371, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 438.487850189209, 'used': 18.224109649658203}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.3287239074707}} +2025-01-02 02:19:28,317 INFO HandlerThread:2087140 [system_monitor.py:probe():224] Finished collecting system info +2025-01-02 02:19:28,317 INFO HandlerThread:2087140 [system_monitor.py:probe():227] Publishing system info +2025-01-02 02:19:28,317 DEBUG HandlerThread:2087140 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2025-01-02 02:19:29,117 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/conda-environment.yaml +2025-01-02 02:19:34,744 DEBUG HandlerThread:2087140 [system_info.py:_save_conda():224] Saving conda packages done +2025-01-02 02:19:34,747 INFO HandlerThread:2087140 [system_monitor.py:probe():229] Finished publishing system info +2025-01-02 02:19:34,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:19:34,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: keepalive +2025-01-02 02:19:34,755 DEBUG SenderThread:2087140 [sender.py:send():379] send: files +2025-01-02 02:19:34,755 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-02 02:19:35,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: python_packages +2025-01-02 02:19:35,036 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: python_packages +2025-01-02 02:19:35,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:19:35,038 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:19:35,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:35,094 INFO wandb-upload_0:2087140 [upload_job.py:push():130] Uploaded file /tmp/tmpxepl3mlrwandb/sl1d2k9b-wandb-metadata.json +2025-01-02 02:19:35,118 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/conda-environment.yaml +2025-01-02 02:19:35,119 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-metadata.json +2025-01-02 02:19:35,119 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/requirements.txt +2025-01-02 02:19:35,205 DEBUG SenderThread:2087140 [sender.py:send():379] send: telemetry +2025-01-02 02:19:35,205 DEBUG SenderThread:2087140 [sender.py:send():379] send: config +2025-01-02 02:19:35,205 DEBUG SenderThread:2087140 [sender.py:send():379] send: telemetry +2025-01-02 02:19:35,205 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:19:35,206 DEBUG SenderThread:2087140 [sender.py:send():379] send: telemetry +2025-01-02 02:19:35,206 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:19:35,206 WARNING SenderThread:2087140 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) +2025-01-02 02:19:35,206 DEBUG SenderThread:2087140 [sender.py:send():379] send: telemetry +2025-01-02 02:19:35,206 DEBUG SenderThread:2087140 [sender.py:send():379] send: telemetry +2025-01-02 02:19:35,206 DEBUG SenderThread:2087140 [sender.py:send():379] send: config +2025-01-02 02:19:36,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:36,120 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:19:37,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:38,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:38,122 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:19:38,212 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:19:39,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:40,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:41,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:42,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:43,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:44,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:19:44,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:45,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:46,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:47,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:48,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:49,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:19:49,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:50,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:19:50,037 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:19:50,076 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:51,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:52,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:52,131 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:19:53,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:54,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:54,484 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:19:55,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:56,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:57,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:58,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:19:59,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:00,045 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:00,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:01,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:01,136 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/config.yaml +2025-01-02 02:20:02,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:03,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:04,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:05,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:20:05,037 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:20:05,076 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:06,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:06,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:07,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:08,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:09,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:10,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:11,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:12,044 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:12,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:13,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:14,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:15,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:16,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:17,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:17,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:18,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:19,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:19,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:20:19,437 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:20:19,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:20:19,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:20:19,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:20:19,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: metric +2025-01-02 02:20:19,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:20:19,439 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:20:19,444 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:20:20,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:20:20,039 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:20:20,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:20,159 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:20:21,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:22,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:22,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:22,160 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:20:23,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:24,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:25,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:26,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:27,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:28,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:28,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:28,242 DEBUG SystemMonitor:2087140 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-02 02:20:28,245 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:20:29,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:30,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:31,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:32,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:33,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:34,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:34,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:35,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:20:35,037 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:20:35,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:35,194 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/config.yaml +2025-01-02 02:20:36,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:37,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:38,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:39,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:40,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:40,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:41,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:42,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:43,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:44,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:45,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:45,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:46,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:47,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:48,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:49,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:50,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:20:50,038 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:20:50,079 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:50,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:50,986 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:20:50,987 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:20:50,987 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:20:50,990 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:20:51,212 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:20:51,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:52,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:52,213 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:20:53,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:54,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:55,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:56,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:20:56,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:57,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:58,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:20:58,247 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:20:59,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:00,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:01,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:02,044 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:02,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:03,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:04,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:05,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:21:05,038 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:21:05,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:06,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:07,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:07,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:08,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:09,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:10,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:11,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:12,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:13,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:13,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:14,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:15,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:16,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:17,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:18,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:18,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:19,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:20,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:21:20,038 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:21:20,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:21,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:21,666 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:21:21,667 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:21:21,667 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:21:21,670 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:21:22,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:22,241 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:21:23,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:23,708 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:24,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:24,243 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:21:25,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:26,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:27,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:28,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:28,250 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:21:29,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:29,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:30,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:31,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:32,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:33,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:34,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:35,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:35,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:21:35,170 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:21:35,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:36,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:37,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:38,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:39,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:40,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:40,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:41,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:42,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:43,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:44,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:45,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:46,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:46,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:47,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:48,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:49,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:50,179 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:50,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:21:50,183 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:21:51,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:51,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:52,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:52,330 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:21:52,332 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:21:52,333 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:21:52,338 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:21:53,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:53,269 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:21:54,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:54,270 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:21:55,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:56,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:57,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:21:57,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:58,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:21:58,254 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:21:59,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:00,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:01,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:02,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:03,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:03,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:04,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:05,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:05,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:22:05,175 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:22:06,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:07,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:08,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:08,304 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:09,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:10,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:11,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:12,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:13,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:14,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:14,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:15,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:16,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:17,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:18,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:19,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:19,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:20,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:22:20,181 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:22:20,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:21,185 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:22,067 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:22:22,068 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:22:22,069 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:22:22,073 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:22:22,302 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:22:22,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:23,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:24,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:24,303 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:22:25,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:25,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:26,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:27,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:28,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:28,258 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:22:29,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:30,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:30,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:31,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:32,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:33,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:34,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:35,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:35,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:22:35,177 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:22:35,217 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:36,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:37,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:38,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:39,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:40,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:41,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:41,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:42,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:43,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:44,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:45,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:46,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:46,224 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:47,224 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:48,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:49,223 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:50,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:22:50,182 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:22:50,224 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:50,884 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:22:50,885 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:22:50,885 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:22:50,887 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:22:51,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:51,330 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:22:51,925 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:52,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:52,331 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:22:53,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:54,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:55,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:56,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:57,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:22:57,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:58,262 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:22:58,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:22:59,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:00,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:01,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:02,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:02,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:03,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:04,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:05,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:23:05,178 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:23:05,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:06,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:07,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:07,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:08,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:09,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:10,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:11,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:12,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:13,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:13,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:14,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:15,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:16,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:17,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:18,179 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:18,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:19,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:19,708 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:23:19,710 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:23:19,710 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:23:19,715 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:23:20,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:23:20,179 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:23:20,363 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:23:20,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:21,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:22,365 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:23:22,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:23,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:23,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:24,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:25,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:26,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:27,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:28,266 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:23:28,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:29,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:29,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:30,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:31,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:32,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:33,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:34,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:35,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:23:35,179 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:23:35,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:35,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:36,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:37,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:38,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:39,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:40,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:41,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:41,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:42,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:43,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:44,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:45,490 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:46,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:46,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:47,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:48,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:23:48,417 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:23:48,417 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:23:48,423 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:23:48,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:49,397 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:23:49,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:50,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:23:50,179 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:23:50,398 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:23:50,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:51,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:51,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:52,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:53,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:54,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:55,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:56,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:57,185 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:23:57,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:58,270 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:23:58,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:23:59,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:00,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:01,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:02,986 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:03,185 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:03,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:04,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:05,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:24:05,179 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:24:05,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:06,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:07,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:08,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:08,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:09,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:10,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:11,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:12,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:13,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:14,186 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:14,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:15,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:16,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:17,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:18,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:19,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:19,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:20,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:24:20,108 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:24:20,109 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:24:20,113 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:24:20,433 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:24:20,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:24:20,514 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:24:20,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:21,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:22,435 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:24:22,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:23,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:24,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:24,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:25,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:26,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:27,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:28,274 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:24:28,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:29,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:30,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:30,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:31,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:32,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:33,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:34,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:35,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:24:35,254 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:24:35,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:35,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:36,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:37,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:38,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:39,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:40,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:41,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:41,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:42,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:43,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:44,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:45,918 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:46,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:46,917 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:47,917 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:48,918 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:49,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:50,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:24:50,257 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:24:50,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:51,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:24:51,348 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:24:51,348 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:24:51,349 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:51,352 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:24:51,464 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:24:51,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:52,465 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:24:52,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:53,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:54,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:55,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:56,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:57,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:24:57,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:58,278 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:24:58,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:24:59,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:00,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:01,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:02,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:03,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:03,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:04,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:05,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:25:05,254 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:25:05,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:06,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:07,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:08,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:08,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:09,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:10,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:11,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:12,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:13,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:14,256 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:14,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:15,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:16,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:17,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:18,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:19,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:19,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:20,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:25:20,261 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:25:20,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:21,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:25:21,487 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:25:21,487 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:25:21,493 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:25:21,494 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:25:21,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:22,495 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:25:22,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:23,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:24,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:24,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:25,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:26,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:27,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:28,282 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:25:28,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:29,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:30,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:30,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:31,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:32,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:33,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:34,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:35,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:25:35,259 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:25:35,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:35,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:36,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:37,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:38,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:39,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:40,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:41,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:41,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:42,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:43,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:44,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:45,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:46,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:46,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:47,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:48,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:49,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:50,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:25:50,268 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:25:50,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:51,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:51,490 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:25:51,492 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:25:51,493 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:25:51,498 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:25:51,526 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:25:51,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:52,527 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:25:52,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:53,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:54,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:55,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:56,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:57,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:25:57,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:58,286 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:25:58,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:25:59,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:00,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:01,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:02,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:02,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:03,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:04,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:05,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:26:05,263 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:26:05,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:06,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:07,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:07,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:08,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:09,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:10,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:11,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:12,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:13,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:13,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:14,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:15,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:16,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:17,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:18,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:19,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:19,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:20,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:26:20,264 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:26:20,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:21,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:22,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:23,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:24,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:24,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:25,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:26,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:26:26,200 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:26:26,201 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:26:26,206 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:26:26,558 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:26:26,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:27,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:28,290 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:26:28,560 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:26:28,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:29,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:30,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:30,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:31,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:32,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:33,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:34,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:35,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:26:35,264 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:26:35,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:35,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:36,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:37,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:38,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:39,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:40,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:41,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:41,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:42,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:43,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:44,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:45,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:46,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:47,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:47,974 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:48,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:49,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:50,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:26:50,272 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:26:50,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:51,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:52,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:52,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:53,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:26:53,600 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:26:53,601 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:26:53,602 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:26:54,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:54,596 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:26:54,597 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:26:55,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:56,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:57,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:58,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:26:58,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:26:58,294 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:26:59,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:00,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:01,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:02,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:03,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:03,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:04,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:05,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:05,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:27:05,265 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:27:06,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:07,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:08,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:08,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:09,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:10,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:11,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:12,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:13,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:14,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:14,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:15,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:16,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:17,066 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:18,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:19,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:19,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:20,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:20,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:27:20,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:27:21,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:22,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:22,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:27:22,812 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:27:22,813 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:27:22,814 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:27:23,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:23,636 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:27:24,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:24,637 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:27:24,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:25,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:26,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:27,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:28,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:28,299 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:27:29,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:30,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:30,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:31,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:32,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:33,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:34,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:35,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:35,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:27:35,267 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:27:35,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:36,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:37,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:38,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:39,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:40,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:41,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:41,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:42,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:43,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:44,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:45,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:46,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:47,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:47,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:48,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:49,223 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:50,228 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:50,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:27:50,280 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:27:51,228 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:52,228 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:52,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:53,228 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:54,228 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:54,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:27:54,977 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:27:54,978 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:27:54,980 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:27:55,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:55,666 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:27:56,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:56,667 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:27:57,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:58,302 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:27:58,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:27:58,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:27:59,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:00,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:01,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:02,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:03,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:03,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:04,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:05,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:28:05,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:28:05,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:06,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:07,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:08,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:09,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:09,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:10,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:11,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:12,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:13,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:14,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:14,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:15,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:16,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:17,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:18,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:19,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:20,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:28:20,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:28:20,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:20,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:21,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:22,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:23,384 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:24,387 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:25,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:26,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:26,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:27,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:28,304 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:28:28,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:28,662 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:28:28,664 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:28:28,664 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:28:28,666 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:28:28,691 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:28:29,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:30,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:30,692 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:28:31,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:32,384 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:32,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:33,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:34,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:35,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:28:35,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:28:35,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:36,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:37,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:37,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:38,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:39,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:40,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:41,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:42,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:43,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:43,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:44,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:45,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:46,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:47,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:48,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:49,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:49,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:50,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:28:50,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:28:50,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:51,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:52,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:53,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:54,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:28:54,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:55,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:56,411 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:57,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:57,726 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:28:57,727 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:28:57,728 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:28:57,729 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:28:58,308 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:28:58,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:28:58,724 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:28:58,724 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:28:59,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:00,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:00,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:01,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:02,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:03,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:04,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:05,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:29:05,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:29:05,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:05,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:06,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:07,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:08,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:09,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:10,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:11,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:11,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:12,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:13,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:14,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:15,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:16,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:16,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:17,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:18,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:19,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:20,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:29:20,273 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:29:20,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:21,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:21,442 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:22,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:23,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:24,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:25,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:26,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:26,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:29:26,844 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:29:26,845 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:29:26,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:26,846 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:29:27,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:27,752 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:29:28,311 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:29:28,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:28,753 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:29:29,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:30,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:31,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:32,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:32,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:33,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:34,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:35,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:29:35,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:29:35,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:36,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:37,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:38,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:38,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:39,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:40,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:41,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:42,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:43,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:43,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:44,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:45,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:46,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:47,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:48,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:49,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:49,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:50,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:29:50,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:29:50,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:51,442 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:52,444 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:53,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:54,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:29:54,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:55,449 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:56,337 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:29:56,339 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:29:56,340 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:29:56,341 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:29:56,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:56,784 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:29:57,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:58,315 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:29:58,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:29:58,786 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:29:59,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:00,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:00,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:01,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:02,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:03,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:04,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:05,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:30:05,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:30:05,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:05,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:06,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:07,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:08,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:09,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:10,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:11,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:11,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:12,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:13,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:14,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:15,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:16,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:17,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:17,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:18,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:19,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:20,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:30:20,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:30:20,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:21,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:22,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:22,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:23,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:24,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:25,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:26,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:27,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:28,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:30:28,144 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:30:28,145 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:30:28,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:28,146 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:30:28,319 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:30:28,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:28,812 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:30:29,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:30,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:30,814 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:30:31,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:32,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:33,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:33,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:34,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:35,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:30:35,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:30:35,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:36,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:37,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:38,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:38,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:39,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:40,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:41,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:42,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:43,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:44,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:44,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:45,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:46,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:47,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:48,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:49,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:49,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:50,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:30:50,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:30:50,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:51,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:52,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:53,779 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:54,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:54,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:55,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:56,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:57,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:58,320 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:30:58,685 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:30:58,686 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:30:58,686 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:30:58,688 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:30:58,844 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:30:59,086 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:30:59,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:30:59,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:00,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:00,846 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:31:01,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:02,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:03,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:04,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:05,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:31:05,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:31:05,395 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:05,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:06,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:07,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:08,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:09,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:10,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:11,205 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:11,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:12,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:13,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:14,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:15,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:15,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:16,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:17,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:18,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:19,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:20,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:31:20,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:31:20,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:21,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:21,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:22,941 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:23,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:24,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:25,950 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:26,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:26,949 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:27,953 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:28,322 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:31:28,795 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:31:28,796 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:31:28,796 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:31:28,797 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:31:28,879 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:31:29,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:30,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:30,881 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:31:31,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:32,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:32,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:33,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:34,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:35,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:35,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:31:35,274 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:31:36,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:37,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:38,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:38,426 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:39,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:40,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:41,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:42,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:43,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:43,508 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:44,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:45,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:46,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:47,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:48,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:49,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:49,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:50,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:31:50,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:31:50,501 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:51,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:52,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:53,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:54,497 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:54,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:31:55,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:56,066 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:31:56,067 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:31:56,067 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:31:56,069 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:31:56,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:56,913 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:31:56,913 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:31:57,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:58,326 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:31:58,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:31:59,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:00,328 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:00,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:01,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:02,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:03,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:04,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:05,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:32:05,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:32:05,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:05,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:06,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:07,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:08,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:09,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:10,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:10,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:11,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:12,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:13,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:14,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:15,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:15,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:16,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:17,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:18,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:19,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:20,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:32:20,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:32:20,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:21,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:21,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:22,924 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:23,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:24,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:25,926 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:26,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:26,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:27,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:28,331 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:32:28,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:32:28,518 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:32:28,519 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:32:28,520 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:32:28,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:28,944 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:32:29,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:30,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:30,946 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:32:31,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:32,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:32,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:33,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:34,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:35,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:32:35,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:32:35,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:36,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:37,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:38,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:38,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:39,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:40,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:41,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:42,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:43,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:44,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:44,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:45,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:46,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:47,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:48,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:49,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:49,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:50,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:32:50,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:32:50,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:51,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:52,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:53,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:54,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:55,645 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:32:55,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:56,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:57,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:58,340 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:32:58,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:32:59,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:00,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:33:00,454 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:33:00,455 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:33:00,456 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:33:00,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:00,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:00,984 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:33:01,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:02,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:02,986 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:33:03,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:04,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:05,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:33:05,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:33:05,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:06,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:06,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:07,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:08,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:09,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:10,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:11,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:11,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:12,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:13,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:14,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:15,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:16,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:16,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:17,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:18,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:19,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:20,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:33:20,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:33:20,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:21,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:22,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:22,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:23,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:24,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:25,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:26,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:27,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:27,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:28,340 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:33:28,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:29,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:30,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:33:30,818 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:33:30,818 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:33:30,819 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:33:31,012 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:33:31,223 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:31,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:32,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:33,014 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:33:33,225 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:33,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:34,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:35,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:33:35,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:33:35,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:36,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:37,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:38,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:39,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:39,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:40,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:41,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:42,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:43,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:44,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:44,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:45,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:46,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:47,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:48,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:49,222 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:49,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:50,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:33:50,275 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:33:50,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:51,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:52,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:53,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:54,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:55,226 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:33:55,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:56,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:57,966 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:58,351 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:33:58,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:33:59,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:00,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:00,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:34:00,848 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:34:00,849 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:34:00,851 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:34:01,042 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:34:01,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:02,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:03,043 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:34:03,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:04,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:05,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:05,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:34:05,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:34:05,389 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:06,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:07,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:08,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:09,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:10,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:11,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:11,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:12,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:13,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:14,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:15,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:16,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:16,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:17,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:18,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:19,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:20,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:20,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:34:20,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:34:21,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:22,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:22,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:23,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:24,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:25,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:26,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:27,257 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:28,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:28,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:28,347 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:34:29,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:30,215 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:34:30,216 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:34:30,216 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:34:30,219 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:34:30,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:31,069 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:34:31,069 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:34:31,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:32,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:33,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:33,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:34,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:35,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:34:35,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:34:35,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:36,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:37,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:38,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:39,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:39,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:40,360 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:41,360 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:42,360 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:43,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:44,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:44,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:45,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:46,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:47,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:48,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:49,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:50,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:50,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:34:50,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:34:50,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:51,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:52,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:53,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:54,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:55,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:34:55,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:56,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:57,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:58,357 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:34:58,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:34:59,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:00,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:00,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:00,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:35:00,693 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:35:00,694 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:35:00,697 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:35:01,093 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:35:01,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:02,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:03,095 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:35:03,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:04,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:05,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:35:05,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:35:05,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:05,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:06,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:07,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:08,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:09,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:10,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:11,257 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:11,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:12,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:13,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:14,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:15,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:16,257 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:16,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:17,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:18,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:19,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:20,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:35:20,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:35:20,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:21,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:22,257 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:22,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:23,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:24,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:25,448 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:26,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:27,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:27,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:28,359 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:35:28,445 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:29,350 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:35:29,351 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:35:29,351 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:35:29,352 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:35:29,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:30,122 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:35:30,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:31,123 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:35:31,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:32,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:33,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:33,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:34,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:35,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:35:35,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:35:35,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:36,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:37,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:38,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:39,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:39,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:40,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:41,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:42,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:43,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:44,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:44,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:45,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:46,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:47,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:48,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:49,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:49,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:50,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:35:50,276 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:35:50,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:51,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:52,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:53,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:54,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:55,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:35:55,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:56,539 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:57,540 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:58,369 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:35:58,538 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:35:59,538 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:00,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:00,538 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:01,539 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:01,590 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:36:01,592 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:36:01,592 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:36:01,594 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:36:02,157 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:36:02,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:03,158 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:36:03,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:04,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:05,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:36:05,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:36:05,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:05,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:06,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:07,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:08,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:09,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:10,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:11,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:11,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:12,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:13,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:14,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:15,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:16,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:16,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:17,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:18,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:19,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:20,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:36:20,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:36:20,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:21,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:22,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:22,546 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:23,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:24,554 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:25,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:26,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:27,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:27,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:28,366 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:36:28,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:29,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:36:29,481 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:36:29,481 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:36:29,483 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:36:29,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:30,182 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:36:30,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:31,183 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:36:31,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:32,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:32,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:33,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:34,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:35,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:36:35,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:36:35,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:36,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:37,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:38,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:38,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:39,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:40,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:41,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:42,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:43,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:44,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:44,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:45,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:46,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:47,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:48,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:49,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:50,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:50,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:36:50,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:36:50,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:51,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:52,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:53,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:54,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:55,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:36:55,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:56,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:57,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:58,373 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:36:58,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:36:59,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:00,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:00,580 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:37:00,581 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:37:00,581 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:37:00,582 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:37:00,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:01,219 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:37:01,219 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:37:01,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:02,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:03,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:04,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:05,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:37:05,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:37:05,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:06,271 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:06,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:07,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:08,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:09,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:10,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:11,271 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:11,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:12,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:13,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:14,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:15,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:16,271 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:16,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:17,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:18,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:19,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:20,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:37:20,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:37:20,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:21,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:22,271 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:22,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:23,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:24,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:25,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:26,987 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:27,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:27,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:28,387 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:37:28,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:29,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:30,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:31,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:37:31,781 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:37:31,781 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:37:31,783 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:37:32,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:32,255 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:37:32,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:33,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:33,256 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:37:34,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:35,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:35,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:37:35,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:37:36,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:37,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:38,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:38,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:39,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:40,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:41,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:42,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:43,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:44,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:44,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:45,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:46,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:47,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:48,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:49,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:49,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:50,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:50,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:37:50,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:37:51,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:52,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:53,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:54,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:55,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:55,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:37:56,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:57,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:58,198 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:37:58,381 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:37:59,201 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:00,198 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:00,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:01,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:02,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:38:02,112 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:38:02,112 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:38:02,114 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:38:02,279 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:38:02,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:03,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:03,279 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:38:04,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:05,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:05,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:38:05,277 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:38:05,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:06,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:07,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:08,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:09,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:10,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:11,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:11,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:12,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:13,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:14,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:15,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:16,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:16,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:17,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:18,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:19,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:20,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:20,277 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:38:20,278 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:38:21,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:21,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:22,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:23,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:24,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:25,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:26,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:27,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:27,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:28,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:28,381 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:38:29,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:30,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:31,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:32,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:32,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:33,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:34,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:35,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:38:35,023 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:38:35,024 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:38:35,029 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:38:35,308 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:38:35,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:35,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:38:35,430 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:38:36,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:37,309 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:38:37,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:38,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:38,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:39,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:40,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:41,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:42,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:43,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:44,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:44,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:45,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:46,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:47,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:48,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:49,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:50,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:50,426 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:38:50,427 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:38:50,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:51,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:52,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:53,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:54,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:55,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:38:55,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:56,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:57,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:58,385 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:38:58,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:38:59,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:00,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:00,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:01,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:02,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:03,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:04,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:05,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:39:05,429 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:39:05,473 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:05,516 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:06,013 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:39:06,014 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:39:06,014 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:39:06,017 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:39:06,344 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:39:06,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:07,345 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:39:07,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:08,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:09,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:10,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:11,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:11,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:12,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:13,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:14,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:15,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:16,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:17,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:17,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:18,469 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:19,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:20,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:39:20,430 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:39:20,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:21,465 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:22,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:22,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:23,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:24,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:25,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:26,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:27,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:28,389 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:39:28,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:28,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:29,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:30,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:31,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:32,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:33,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:33,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:34,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:35,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:39:35,176 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:39:35,176 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:39:35,177 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:39:35,373 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:39:35,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:35,629 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:39:35,630 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:39:36,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:37,374 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:39:37,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:38,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:39,422 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:39,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:40,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:41,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:42,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:43,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:44,422 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:44,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:45,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:46,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:47,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:48,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:49,422 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:49,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:50,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:39:50,625 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:39:50,665 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:51,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:52,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:53,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:54,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:55,422 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:39:55,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:56,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:57,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:58,393 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:39:58,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:39:59,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:00,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:01,396 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:01,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:02,632 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:03,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:04,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:05,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:40:05,635 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:40:05,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:06,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:06,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:07,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:40:07,172 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:40:07,173 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:40:07,175 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:40:07,401 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:40:07,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:08,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:09,403 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:40:09,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:10,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:11,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:11,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:12,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:13,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:14,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:15,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:16,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:17,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:17,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:18,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:19,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:20,628 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:40:20,629 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:40:20,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:21,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:22,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:22,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:23,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:24,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:25,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:26,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:27,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:28,398 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:40:28,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:28,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:29,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:30,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:31,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:32,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:33,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:33,644 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:34,645 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:35,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:40:35,634 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:40:35,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:36,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:37,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:38,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:38,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:39,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:40:39,040 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:40:39,041 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:40:39,042 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:40:39,430 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:40:39,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:40,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:41,432 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:40:41,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:42,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:43,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:44,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:44,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:45,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:46,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:47,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:48,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:49,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:50,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:50,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:40:50,634 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:40:50,674 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:51,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:52,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:53,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:54,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:55,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:40:55,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:56,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:57,651 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:58,402 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:40:58,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:40:59,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:00,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:01,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:01,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:02,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:03,655 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:04,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:05,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:41:05,643 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:41:05,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:06,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:06,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:07,655 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:08,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:41:08,250 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:41:08,251 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:41:08,253 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:41:08,464 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:41:08,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:09,465 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:41:09,656 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:10,656 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:11,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:11,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:12,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:13,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:14,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:15,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:16,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:17,432 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:17,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:18,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:19,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:20,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:41:20,637 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:41:20,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:21,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:22,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:23,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:23,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:24,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:25,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:26,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:27,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:28,404 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:41:28,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:29,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:29,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:30,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:31,693 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:32,693 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:33,693 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:34,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:34,697 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:35,645 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:41:35,646 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:41:35,697 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:36,694 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:37,694 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:38,694 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:39,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:39,695 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:39,763 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:41:39,765 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:41:39,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:41:39,767 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:41:40,493 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:41:40,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:41,494 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:41:41,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:42,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:43,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:44,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:45,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:45,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:46,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:47,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:48,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:49,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:50,643 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:41:50,644 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:41:50,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:50,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:51,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:52,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:53,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:54,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:55,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:56,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:41:56,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:57,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:58,408 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:41:58,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:41:59,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:00,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:01,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:02,434 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:02,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:03,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:04,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:05,653 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:42:05,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:42:05,722 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:06,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:07,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:07,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:08,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:09,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:10,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:10,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:42:10,729 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:42:10,730 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:42:10,731 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:42:11,526 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:42:11,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:12,526 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:42:12,840 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:12,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:13,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:14,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:15,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:16,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:17,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:18,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:18,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:19,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:20,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:42:20,648 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:42:20,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:21,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:22,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:23,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:24,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:24,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:25,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:26,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:27,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:28,412 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:42:28,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:29,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:30,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:30,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:31,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:32,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:33,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:34,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:35,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:42:35,648 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:42:35,718 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:35,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:36,926 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:37,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:38,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:39,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:40,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:40,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:41,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:42:41,552 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:42:41,552 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:42:41,553 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:42:42,554 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:42:42,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:43,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:44,555 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:42:44,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:45,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:46,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:46,592 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:47,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:48,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:49,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:50,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:50,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:42:50,649 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:42:51,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:51,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:52,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:53,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:54,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:55,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:56,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:57,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:42:57,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:58,416 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:42:58,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:42:59,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:00,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:01,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:02,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:02,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:03,592 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:04,592 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:05,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:05,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:43:05,649 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:43:06,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:07,592 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:07,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:08,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:08,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:43:08,688 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:43:08,689 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:43:08,691 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:43:09,586 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:43:09,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:10,586 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:43:10,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:11,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:12,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:13,589 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:13,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:14,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:15,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:16,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:17,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:18,589 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:18,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:19,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:20,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:20,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:43:20,649 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:43:21,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:22,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:23,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:24,590 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:24,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:25,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:26,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:27,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:28,421 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:43:28,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:29,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:30,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:30,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:31,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:32,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:33,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:34,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:35,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:35,604 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:35,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:43:35,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:43:36,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:37,604 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:38,604 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:39,016 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:43:39,017 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:43:39,017 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:43:39,019 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:43:39,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:39,624 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:43:40,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:40,625 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:43:41,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:41,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:42,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:43,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:44,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:45,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:46,591 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:46,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:47,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:48,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:49,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:50,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:50,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:43:50,657 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:43:51,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:51,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:52,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:53,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:54,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:55,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:56,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:57,591 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:43:57,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:58,425 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:43:58,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:43:59,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:00,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:01,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:02,591 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:02,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:03,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:04,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:05,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:05,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:44:05,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:44:06,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:07,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:07,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:08,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:09,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:10,472 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:44:10,474 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:44:10,475 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:44:10,476 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:44:10,657 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:44:11,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:12,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:12,659 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:44:13,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:13,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:14,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:15,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:16,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:17,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:18,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:18,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:19,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:20,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:20,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:44:20,657 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:44:21,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:22,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:23,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:24,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:24,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:25,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:26,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:27,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:28,429 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:44:28,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:29,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:30,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:30,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:31,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:32,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:33,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:34,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:35,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:35,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:35,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:44:35,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:44:36,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:37,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:38,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:39,532 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:40,532 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:40,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:41,472 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:44:41,473 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:44:41,473 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:44:41,474 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:44:41,685 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:44:42,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:42,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:43,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:44,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:44,689 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:44:45,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:46,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:46,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:47,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:48,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:49,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:50,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:50,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:44:50,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:44:51,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:51,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:52,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:53,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:54,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:55,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:56,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:57,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:44:57,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:58,432 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:44:58,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:44:59,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:00,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:01,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:02,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:03,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:03,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:04,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:05,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:05,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:45:05,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:45:06,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:07,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:08,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:08,628 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:09,629 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:10,628 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:11,575 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:45:11,576 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:45:11,576 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:45:11,577 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:45:11,720 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:45:12,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:13,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:13,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:14,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:14,723 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:45:15,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:16,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:17,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:18,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:19,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:19,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:20,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:20,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:45:20,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:45:21,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:22,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:23,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:24,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:25,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:25,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:26,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:27,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:28,436 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:45:28,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:29,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:30,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:31,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:31,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:32,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:33,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:34,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:35,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:35,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:45:35,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:45:36,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:36,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:37,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:38,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:39,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:40,539 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:41,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:42,542 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:42,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:43,542 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:44,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:44,712 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:45:44,714 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:45:44,714 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:45:44,715 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:45:44,754 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:45:45,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:46,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:46,755 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:45:47,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:48,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:48,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:49,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:50,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:50,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:45:50,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:45:51,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:52,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:53,543 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:54,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:54,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:45:55,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:56,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:57,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:58,440 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:45:58,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:45:59,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:00,442 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:00,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:01,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:02,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:03,544 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:04,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:05,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:05,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:05,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:46:05,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:46:06,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:07,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:08,554 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:09,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:10,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:10,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:11,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:12,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:13,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:13,943 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:46:13,944 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:46:13,945 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:46:13,946 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:46:14,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:14,784 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:46:15,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:15,985 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:16,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:16,785 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:46:17,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:18,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:19,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:20,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:20,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:46:20,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:46:21,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:21,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:22,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:23,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:24,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:25,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:26,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:27,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:27,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:28,445 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:46:28,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:29,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:30,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:31,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:32,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:32,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:33,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:34,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:35,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:35,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:46:35,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:46:36,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:37,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:38,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:38,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:39,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:40,561 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:41,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:42,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:43,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:43,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:44,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:45,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:46,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:46:46,170 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:46:46,171 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:46:46,172 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:46:46,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:46,812 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:46:47,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:48,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:48,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:48,814 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:46:49,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:50,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:50,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:46:50,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:46:51,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:52,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:53,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:54,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:54,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:46:55,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:56,578 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:57,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:58,448 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:46:58,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:46:59,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:00,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:00,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:01,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:02,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:03,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:04,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:05,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:05,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:05,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:47:05,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:47:06,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:07,579 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:08,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:09,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:10,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:10,623 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:11,590 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:12,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:13,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:14,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:47:14,236 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:47:14,236 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:47:14,237 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:47:14,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:14,843 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:47:15,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:16,276 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:16,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:16,845 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:47:17,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:18,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:19,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:20,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:20,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:47:20,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:47:21,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:21,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:22,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:23,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:24,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:25,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:26,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:27,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:27,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:28,452 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:47:28,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:29,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:30,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:31,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:32,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:33,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:33,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:34,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:35,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:35,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:47:35,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:47:36,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:37,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:38,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:39,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:39,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:40,646 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:41,646 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:42,655 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:43,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:44,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:44,652 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:45,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:47:45,153 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:47:45,153 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:47:45,155 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:47:45,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:45,871 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:47:46,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:46,872 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:47:47,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:48,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:49,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:50,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:50,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:50,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:47:50,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:47:51,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:52,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:53,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:54,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:55,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:56,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:47:56,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:57,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:58,456 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:47:58,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:47:59,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:00,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:01,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:02,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:02,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:03,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:04,651 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:05,651 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:05,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:48:05,659 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:48:06,651 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:07,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:07,651 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:08,652 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:09,655 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:10,665 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:11,663 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:12,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:12,665 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:13,663 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:14,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:48:14,543 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:48:14,543 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:48:14,545 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:48:14,901 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:48:14,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:15,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:16,903 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:48:16,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:17,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:18,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:18,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:19,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:20,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:48:20,660 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:48:20,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:21,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:22,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:23,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:24,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:24,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:25,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:26,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:27,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:28,461 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:48:28,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:29,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:30,463 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:30,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:31,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:32,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:33,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:34,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:35,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:35,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:48:35,660 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:48:35,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:36,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:37,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:38,953 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:39,953 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:40,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:40,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:41,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:42,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:43,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:44,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:48:44,735 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:48:44,736 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:48:44,737 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:48:44,929 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:48:45,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:45,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:46,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:46,931 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:48:47,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:48,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:49,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:50,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:50,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:48:50,660 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:48:50,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:51,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:52,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:53,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:54,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:55,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:56,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:56,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:48:57,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:58,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:48:58,465 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:48:59,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:00,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:01,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:02,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:02,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:03,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:04,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:05,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:05,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:49:05,669 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:49:06,149 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:07,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:07,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:08,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:09,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:10,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:10,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:49:10,237 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:49:10,237 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:49:10,239 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:49:10,964 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:49:11,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:12,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:12,632 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:12,966 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:49:13,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:14,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:15,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:16,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:17,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:18,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:18,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:19,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:20,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:20,662 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:49:20,662 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:49:21,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:22,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:23,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:23,632 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:24,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:25,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:26,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:27,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:28,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:28,469 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:49:29,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:29,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:30,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:31,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:32,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:33,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:34,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:34,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:35,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:35,666 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:49:35,671 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:49:36,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:37,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:38,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:39,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:39,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:49:39,378 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:49:39,379 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:49:39,380 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:49:39,994 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:49:40,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:40,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:40,994 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:49:41,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:42,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:43,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:44,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:45,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:45,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:46,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:47,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:48,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:49,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:50,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:50,666 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:49:50,666 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:49:50,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:51,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:52,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:53,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:54,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:55,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:56,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:56,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:49:57,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:58,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:49:58,473 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:49:59,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:00,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:01,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:02,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:02,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:03,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:04,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:05,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:05,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:50:05,679 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:50:06,203 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:07,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:07,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:08,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:09,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:10,197 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:11,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:50:11,042 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:50:11,043 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:50:11,044 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:50:12,026 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:50:12,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:12,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:13,027 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:50:13,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:14,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:15,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:16,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:17,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:17,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:18,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:19,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:20,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:20,671 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:50:20,671 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:50:21,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:22,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:23,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:23,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:24,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:25,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:26,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:27,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:28,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:28,477 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:50:29,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:29,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:30,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:31,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:32,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:33,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:34,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:34,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:35,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:35,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:50:35,675 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:50:36,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:37,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:38,211 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:39,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:39,654 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:40,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:41,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:41,349 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:50:41,350 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:50:41,351 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:50:41,352 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:50:42,052 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:50:42,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:43,053 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:50:43,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:44,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:44,684 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:45,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:46,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:47,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:48,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:49,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:50,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:50:50,675 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:50:50,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:50,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:51,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:52,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:53,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:54,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:55,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:56,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:50:56,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:57,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:58,479 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:50:58,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:50:59,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:00,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:01,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:02,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:02,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:03,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:04,683 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:05,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:51:05,675 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:51:05,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:06,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:07,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:07,698 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:08,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:09,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:10,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:11,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:11,986 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:51:11,988 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:51:11,988 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:51:11,989 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:51:12,080 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:51:12,688 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:12,688 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:13,081 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:51:13,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:14,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:15,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:16,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:17,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:18,687 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:18,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:19,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:20,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:51:20,676 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:51:20,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:21,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:22,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:23,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:24,688 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:24,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:25,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:26,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:27,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:28,485 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:51:28,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:29,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:30,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:30,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:31,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:32,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:33,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:34,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:35,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:51:35,676 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:51:35,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:35,840 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:36,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:37,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:38,788 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:39,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:40,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:41,699 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:41,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:42,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:51:42,246 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:51:42,246 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:51:42,247 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:51:42,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:43,111 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:51:43,112 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:51:43,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:44,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:45,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:46,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:46,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:47,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:48,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:49,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:50,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:51:50,676 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:51:50,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:51,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:51,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:52,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:53,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:54,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:55,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:56,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:57,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:51:57,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:58,489 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:51:58,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:51:59,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:00,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:01,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:02,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:03,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:03,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:04,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:05,680 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:52:05,680 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:52:05,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:06,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:07,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:08,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:08,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:09,334 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:52:09,335 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:52:09,336 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:52:09,338 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:52:09,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:10,142 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:52:10,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:11,143 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:52:11,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:12,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:13,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:14,731 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:14,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:15,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:16,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:17,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:18,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:19,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:20,680 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:52:20,680 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:52:20,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:20,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:21,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:22,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:23,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:24,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:25,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:26,732 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:26,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:27,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:28,492 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:52:28,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:29,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:30,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:31,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:32,495 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:32,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:33,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:34,868 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:35,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:52:35,681 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:52:35,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:36,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:37,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:37,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:37,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:52:37,868 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:52:37,869 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:52:37,870 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:52:38,174 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:52:38,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:39,176 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:52:39,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:40,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:41,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:42,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:43,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:43,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:44,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:45,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:46,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:47,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:48,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:49,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:49,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:50,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:52:50,682 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:52:50,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:51,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:52,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:53,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:54,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:52:54,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:55,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:56,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:57,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:58,496 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:52:58,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:52:59,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:00,498 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:00,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:01,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:02,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:03,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:04,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:05,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:53:05,681 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:53:05,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:05,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:06,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:53:06,624 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:53:06,624 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:53:06,626 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:53:07,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:07,199 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:53:08,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:09,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:09,200 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:53:10,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:11,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:11,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:12,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:13,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:14,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:15,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:16,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:17,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:17,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:18,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:19,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:20,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:20,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:53:20,682 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:53:21,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:22,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:22,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:23,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:24,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:25,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:26,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:27,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:28,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:28,500 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:53:28,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:29,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:30,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:31,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:32,030 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:33,030 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:33,749 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:34,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:35,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:35,685 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:53:35,691 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:53:36,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:37,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:38,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:38,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:39,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:39,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:53:39,114 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:53:39,114 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:53:39,115 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:53:39,225 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:53:40,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:41,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:41,227 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:53:42,045 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:43,045 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:44,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:44,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:45,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:46,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:47,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:48,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:49,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:50,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:50,685 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:53:50,685 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:53:50,832 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:51,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:52,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:53,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:54,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:55,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:56,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:56,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:53:57,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:58,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:53:58,504 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:53:59,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:00,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:01,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:02,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:02,507 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:03,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:04,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:05,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:05,688 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:54:05,688 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:54:06,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:07,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:07,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:08,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:08,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:54:08,291 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:54:08,291 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:54:08,293 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:54:09,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:09,256 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:54:09,256 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:54:10,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:11,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:12,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:12,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:13,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:14,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:15,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:16,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:17,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:18,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:18,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:19,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:20,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:20,688 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:54:20,688 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:54:21,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:22,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:23,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:23,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:24,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:25,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:26,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:27,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:28,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:28,509 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:54:29,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:29,511 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:30,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:31,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:32,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:33,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:34,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:34,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:35,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:35,693 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:54:35,694 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:54:36,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:37,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:38,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:39,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:39,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:39,778 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:54:39,779 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:54:39,779 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:54:39,780 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:54:40,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:40,285 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:54:41,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:41,286 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:54:42,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:43,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:44,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:45,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:45,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:46,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:47,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:48,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:49,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:50,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:50,692 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:54:50,692 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:54:50,815 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:51,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:52,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:53,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:54,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:55,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:56,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:56,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:54:57,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:58,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:54:58,513 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:54:59,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:00,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:01,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:02,189 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:02,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:03,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:04,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:05,198 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:05,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:55:05,706 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:55:06,196 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:07,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:07,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:08,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:09,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:55:09,158 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:55:09,158 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:55:09,159 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:55:09,313 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:55:09,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:10,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:11,315 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:55:11,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:12,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:13,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:13,761 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:14,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:15,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:16,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:17,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:18,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:19,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:19,761 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:20,722 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:55:20,723 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:55:20,762 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:21,720 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:22,720 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:23,720 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:24,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:25,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:25,761 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:26,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:27,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:28,515 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:55:28,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:29,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:30,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:31,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:31,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:32,731 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:33,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:34,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:35,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:55:35,724 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:55:35,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:36,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:36,858 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:37,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:37,977 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:55:37,979 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:55:37,979 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:55:37,981 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:55:38,343 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:55:38,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:39,344 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:55:39,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:40,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:41,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:42,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:42,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:43,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:44,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:45,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:46,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:47,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:47,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:48,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:49,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:50,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:55:50,725 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:55:50,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:51,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:52,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:52,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:53,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:54,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:55,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:56,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:57,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:58,520 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:55:58,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:55:58,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:55:59,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:00,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:01,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:02,731 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:03,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:03,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:04,740 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:05,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:56:05,734 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:56:05,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:06,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:07,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:08,540 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:56:08,542 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:56:08,542 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:56:08,543 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:56:08,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:09,371 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:56:09,372 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:56:09,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:09,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:10,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:11,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:12,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:13,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:14,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:14,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:15,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:16,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:17,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:18,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:19,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:20,729 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:56:20,729 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:56:20,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:20,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:21,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:22,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:23,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:24,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:25,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:26,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:26,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:27,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:28,524 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:56:28,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:29,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:30,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:31,940 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:32,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:32,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:33,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:34,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:35,736 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:56:35,737 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:56:35,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:36,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:37,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:37,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:38,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:56:38,442 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:56:38,442 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:56:38,444 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:56:38,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:39,403 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:56:39,403 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:56:39,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:40,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:41,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:42,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:42,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:43,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:44,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:45,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:46,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:47,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:48,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:48,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:49,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:50,731 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:56:50,732 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:56:50,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:51,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:52,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:53,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:54,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:56:54,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:55,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:56,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:57,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:58,528 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:56:58,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:56:59,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:00,531 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:00,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:01,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:02,954 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:03,954 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:04,954 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:05,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:57:05,738 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:57:05,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:05,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:06,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:07,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:57:07,221 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:57:07,222 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:57:07,223 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:57:07,431 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:57:07,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:08,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:09,433 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:57:09,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:10,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:11,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:11,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:12,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:13,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:14,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:15,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:16,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:17,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:17,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:18,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:19,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:20,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:57:20,735 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:57:20,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:21,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:22,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:23,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:23,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:24,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:25,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:26,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:27,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:28,532 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:57:28,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:29,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:29,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:30,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:31,965 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:32,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:33,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:34,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:34,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:35,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:57:35,737 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:57:35,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:36,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:57:36,903 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:57:36,903 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:57:36,904 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:57:37,305 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:37,463 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:57:37,463 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:57:38,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:39,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:39,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:40,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:41,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:42,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:43,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:44,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:45,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:45,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:46,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:47,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:48,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:49,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:50,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:50,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:57:50,737 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:57:50,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:51,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:52,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:53,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:54,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:55,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:56,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:56,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:57:57,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:58,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:57:58,536 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:57:59,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:00,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:01,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:02,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:02,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:03,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:04,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:05,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:05,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:58:05,741 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:58:06,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:06,360 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:58:06,362 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:58:06,362 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:58:06,363 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:58:06,490 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:58:07,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:07,491 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:58:07,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:08,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:09,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:10,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:11,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:12,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:13,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:13,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:14,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:15,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:16,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:17,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:18,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:18,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:19,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:20,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:20,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:58:20,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:58:21,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:22,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:23,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:24,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:24,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:25,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:26,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:27,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:28,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:28,540 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:58:29,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:30,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:30,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:31,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:32,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:33,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:34,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:34,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:58:34,623 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:58:34,623 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:58:34,624 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:58:35,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:35,521 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:58:35,522 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:58:35,663 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:35,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:58:35,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:58:36,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:37,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:38,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:39,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:40,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:40,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:41,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:42,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:43,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:44,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:45,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:46,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:46,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:47,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:48,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:49,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:50,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:50,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:58:50,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:58:51,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:51,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:52,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:53,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:54,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:55,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:56,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:57,150 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:57,824 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:58:58,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:58:58,547 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:58:59,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:00,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:01,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:01,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:59:01,852 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:59:01,852 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:59:01,854 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:59:02,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:02,552 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:59:02,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:03,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:03,553 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:59:04,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:05,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:05,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:59:05,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:59:06,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:07,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:08,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:08,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:09,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:10,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:11,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:12,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:13,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:14,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:14,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:15,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:16,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:17,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:18,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:19,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:20,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:20,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:59:20,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:59:20,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:21,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:22,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:23,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:24,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:25,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:26,256 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:26,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:27,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:28,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:28,554 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:59:29,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:30,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:31,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 02:59:31,094 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 02:59:31,094 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 02:59:31,096 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 02:59:31,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:31,575 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 02:59:32,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:32,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:33,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:33,576 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 02:59:34,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:35,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:35,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:59:35,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:59:36,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:37,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:37,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:38,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:39,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:40,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:41,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:42,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:43,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:43,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:44,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:45,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:46,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:47,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:48,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:48,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:49,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:50,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:50,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 02:59:50,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 02:59:51,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:52,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:53,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:54,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:54,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 02:59:55,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:56,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:57,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:58,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 02:59:58,564 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 02:59:59,501 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:00,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:00,570 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:01,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:02,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:03,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:04,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:00:04,209 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:00:04,209 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:00:04,211 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:00:04,605 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:00:04,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:05,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:05,606 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:00:05,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:00:05,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:00:05,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:06,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:07,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:08,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:09,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:10,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:11,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:11,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:12,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:13,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:14,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:15,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:16,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:17,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:17,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:18,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:19,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:20,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:20,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:00:20,742 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:00:21,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:22,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:23,606 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:23,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:24,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:25,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:26,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:27,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:28,560 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:00:28,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:29,564 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:29,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:30,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:31,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:32,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:33,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:00:33,070 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:00:33,070 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:00:33,075 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:00:33,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:33,633 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:00:33,634 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:00:34,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:35,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:35,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:35,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:00:35,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:00:36,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:37,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:38,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:39,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:40,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:40,843 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:41,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:42,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:43,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:44,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:45,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:46,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:46,843 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:47,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:48,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:49,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:50,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:50,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:00:50,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:00:51,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:51,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:52,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:53,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:54,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:55,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:56,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:57,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:57,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:00:58,574 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:00:58,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:00:59,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:00,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:01,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:02,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:02,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:01:02,658 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:01:02,658 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:01:02,660 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:01:02,663 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:01:03,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:03,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:03,664 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:01:04,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:05,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:05,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:01:05,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:01:06,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:07,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:08,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:09,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:09,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:10,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:11,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:12,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:13,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:14,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:14,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:15,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:16,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:17,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:18,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:19,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:19,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:20,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:20,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:01:20,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:01:21,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:22,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:23,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:24,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:25,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:25,637 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:26,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:27,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:28,571 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:01:28,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:29,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:30,573 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:30,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:31,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:01:31,265 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:01:31,266 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:01:31,267 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:01:31,671 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:31,695 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:01:32,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:33,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:33,697 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:01:34,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:35,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:35,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:01:35,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:01:35,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:36,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:37,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:38,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:39,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:40,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:41,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:41,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:42,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:43,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:44,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:45,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:46,227 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:46,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:47,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:48,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:49,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:50,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:50,742 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:01:50,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:01:51,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:51,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:52,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:53,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:54,681 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:55,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:56,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:57,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:01:57,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:58,571 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:01:58,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:01:59,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:00,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:02:00,403 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:02:00,403 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:02:00,405 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:02:00,724 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:02:00,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:01,726 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:02:01,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:02,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:02,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:03,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:04,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:05,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:02:05,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:02:05,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:06,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:07,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:07,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:08,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:09,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:10,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:11,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:12,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:13,139 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:13,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:14,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:15,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:16,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:17,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:18,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:19,139 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:19,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:20,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:02:20,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:02:20,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:21,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:22,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:23,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:24,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:24,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:25,819 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:26,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:27,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:28,573 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:02:28,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:29,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:02:29,025 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:02:29,025 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:02:29,027 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:02:29,750 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:02:29,751 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:02:29,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:30,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:30,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:31,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:32,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:33,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:34,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:35,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:35,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:02:35,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:02:35,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:36,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:37,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:38,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:39,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:40,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:41,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:41,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:42,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:43,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:44,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:45,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:46,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:47,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:47,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:48,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:49,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:50,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:02:50,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:02:50,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:51,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:52,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:52,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:53,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:54,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:55,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:56,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:57,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:58,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:02:58,574 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:02:58,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:02:58,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:02:58,947 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:02:58,947 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:02:58,949 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:02:59,782 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:02:59,782 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:02:59,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:00,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:01,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:02,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:03,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:04,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:04,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:05,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:03:05,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:03:05,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:06,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:07,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:08,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:09,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:09,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:10,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:11,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:12,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:13,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:14,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:15,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:15,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:16,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:17,829 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:18,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:19,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:20,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:03:20,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:03:20,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:20,840 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:21,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:22,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:23,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:24,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:25,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:26,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:26,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:27,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:28,577 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:03:28,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:29,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:30,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:31,179 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:03:31,181 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:03:31,181 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:03:31,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:31,183 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:03:31,811 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:03:31,812 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:03:31,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:32,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:33,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:34,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:35,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:03:35,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:03:35,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:36,820 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:36,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:37,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:38,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:39,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:40,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:41,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:42,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:42,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:43,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:44,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:45,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:46,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:47,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:47,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:48,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:49,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:50,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:03:50,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:03:50,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:51,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:52,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:52,917 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:53,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:54,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:55,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:56,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:57,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:58,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:03:58,578 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:03:58,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:03:59,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:00,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:01,815 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:04:01,816 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:04:01,816 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:04:01,818 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:04:01,836 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:04:02,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:03,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:03,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:03,838 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:04:04,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:05,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:05,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:04:05,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:04:06,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:07,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:08,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:09,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:09,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:10,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:11,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:12,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:13,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:14,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:14,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:15,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:16,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:17,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:18,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:19,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:19,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:20,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:20,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:04:20,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:04:21,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:22,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:23,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:24,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:25,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:25,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:26,222 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:27,222 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:28,230 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:28,589 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:04:29,230 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:30,230 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:30,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:31,230 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:32,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:04:32,079 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:04:32,079 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:04:32,081 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:04:32,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:32,864 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:04:33,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:33,865 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:04:34,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:35,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:35,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:04:35,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:04:35,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:36,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:37,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:38,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:39,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:40,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:41,160 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:41,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:42,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:43,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:44,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:45,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:46,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:47,161 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:47,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:48,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:49,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:50,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:50,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:04:50,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:04:51,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:52,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:52,914 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:53,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:54,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:55,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:56,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:57,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:58,161 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:04:58,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:04:58,585 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:04:59,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:00,238 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:01,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:02,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:03,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:03,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:04,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:04,652 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:05:04,653 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:05:04,654 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:05:04,657 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:05:04,896 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:05:05,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:05,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:05:05,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:05:05,897 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:05:06,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:07,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:08,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:09,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:09,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:10,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:11,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:12,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:13,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:14,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:14,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:15,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:16,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:17,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:18,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:19,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:19,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:20,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:20,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:05:20,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:05:21,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:22,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:23,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:24,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:24,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:25,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:26,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:27,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:28,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:28,590 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:05:29,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:29,593 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:30,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:31,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:32,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:32,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:05:32,674 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:05:32,675 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:05:32,676 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:05:32,921 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:05:33,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:33,922 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:05:34,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:35,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:35,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:35,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:05:35,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:05:36,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:37,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:38,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:39,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:40,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:41,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:41,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:42,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:43,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:44,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:45,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:46,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:47,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:47,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:48,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:49,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:50,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:50,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:05:50,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:05:51,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:52,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:52,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:53,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:54,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:55,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:56,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:57,349 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:58,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:05:58,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:58,595 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:05:59,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:05:59,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:05:59,560 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:05:59,560 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:05:59,561 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:05:59,951 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:06:00,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:00,952 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:06:01,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:02,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:03,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:04,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:04,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:05,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:05,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:06:05,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:06:06,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:07,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:08,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:09,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:10,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:10,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:11,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:12,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:13,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:14,516 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:15,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:16,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:16,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:17,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:18,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:19,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:20,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:20,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:06:20,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:06:21,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:21,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:22,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:23,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:24,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:25,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:26,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:27,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:27,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:28,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:28,600 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:06:29,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:30,441 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:31,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:32,439 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:32,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:33,080 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:06:33,081 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:06:33,081 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:06:33,082 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:06:33,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:33,986 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:06:34,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:34,987 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:06:35,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:35,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:06:35,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:06:36,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:37,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:37,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:38,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:39,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:40,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:41,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:42,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:43,231 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:43,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:44,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:45,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:46,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:47,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:48,231 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:48,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:49,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:50,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:50,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:06:50,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:06:51,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:52,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:53,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:54,231 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:54,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:55,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:56,674 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:57,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:58,604 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:06:58,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:06:59,605 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:06:59,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:00,675 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:01,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:01,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:07:01,782 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:07:01,782 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:07:01,783 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:07:02,024 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:07:02,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:03,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:04,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:05,026 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:07:05,237 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:05,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:05,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:07:05,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:07:06,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:07,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:08,676 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:09,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:10,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:11,237 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:11,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:12,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:13,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:14,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:15,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:16,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:17,237 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:17,677 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:18,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:19,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:20,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:20,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:07:20,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:07:21,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:22,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:22,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:23,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:24,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:25,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:26,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:27,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:28,237 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:28,609 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:07:28,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:29,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:30,686 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:31,686 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:32,686 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:33,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:33,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:34,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:35,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:07:35,255 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:07:35,256 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:07:35,258 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:07:35,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:35,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:07:35,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:07:36,055 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:07:36,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:37,056 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:07:37,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:38,689 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:39,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:39,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:40,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:41,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:42,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:43,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:44,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:45,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:45,690 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:46,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:47,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:48,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:49,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:50,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:50,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:07:50,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:07:50,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:51,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:52,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:53,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:54,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:55,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:56,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:07:56,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:57,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:58,620 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:07:58,805 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:07:59,805 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:00,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:01,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:01,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:02,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:03,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:08:03,704 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:08:03,705 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:08:03,706 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:08:04,087 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:08:04,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:04,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:05,088 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:08:05,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:08:05,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:08:05,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:06,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:06,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:07,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:08,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:09,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:10,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:11,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:12,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:12,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:13,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:14,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:15,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:16,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:17,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:17,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:18,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:19,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:20,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:08:20,750 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:08:20,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:21,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:22,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:22,918 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:23,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:24,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:25,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:26,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:27,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:28,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:28,617 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:08:28,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:29,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:30,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:31,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:31,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:08:31,948 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:08:31,948 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:08:31,951 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:08:32,113 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:08:32,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:33,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:34,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:34,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:35,116 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:08:35,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:08:35,750 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:08:35,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:36,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:37,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:38,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:39,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:40,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:40,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:41,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:42,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:43,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:44,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:45,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:46,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:46,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:47,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:48,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:49,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:50,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:08:50,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:08:50,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:51,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:51,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:52,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:53,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:54,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:55,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:56,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:57,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:08:57,868 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:58,623 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:08:58,868 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:08:59,868 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:00,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:09:00,490 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:09:00,490 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:09:00,492 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:09:00,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:01,144 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:09:01,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:02,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:02,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:03,146 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:09:03,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:04,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:05,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:09:05,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:09:05,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:06,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:07,826 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:07,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:08,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:09,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:10,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:12,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:13,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:13,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:14,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:15,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:16,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:17,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:18,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:19,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:19,254 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:20,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:20,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:09:20,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:09:21,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:22,102 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:23,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:24,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:24,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:25,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:26,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:27,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:27,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:09:27,360 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:09:27,361 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:09:27,362 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:09:28,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:28,173 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:09:28,627 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:09:29,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:29,174 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:09:29,630 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:30,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:31,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:32,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:33,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:34,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:35,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:35,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:35,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:09:35,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:09:36,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:37,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:38,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:39,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:40,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:40,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:41,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:42,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:43,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:44,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:45,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:45,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:46,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:47,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:48,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:49,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:50,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:50,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:09:50,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:09:50,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:51,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:52,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:53,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:54,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:55,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:56,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:56,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:09:57,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:57,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:09:57,342 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:09:57,342 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:09:57,346 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:09:58,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:58,209 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:09:58,631 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:09:59,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:09:59,210 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:10:00,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:01,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:01,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:02,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:03,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:04,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:05,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:05,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:10:05,751 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:10:06,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:06,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:07,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:08,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:09,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:10,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:11,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:12,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:12,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:13,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:14,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:15,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:16,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:17,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:17,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:18,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:19,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:20,159 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:20,761 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:10:20,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:10:21,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:22,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:22,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:23,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:24,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:25,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:26,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:26,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:10:26,309 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:10:26,309 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:10:26,311 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:10:27,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:27,236 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:10:28,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:28,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:28,636 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:10:29,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:29,238 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:10:30,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:31,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:32,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:33,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:34,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:34,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:35,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:35,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:10:35,754 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:10:36,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:37,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:38,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:39,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:40,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:40,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:41,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:42,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:43,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:44,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:45,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:46,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:46,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:47,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:48,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:49,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:50,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:50,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:10:50,754 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:10:51,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:51,884 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:52,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:53,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:54,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:55,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:56,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:56,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:10:56,946 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:10:56,947 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:10:56,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:10:56,949 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:10:57,266 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:10:57,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:58,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:10:58,639 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:10:59,267 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:10:59,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:00,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:01,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:02,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:02,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:03,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:04,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:05,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:05,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:11:05,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:11:06,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:07,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:07,924 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:08,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:09,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:10,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:11,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:12,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:13,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:13,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:14,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:15,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:16,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:17,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:18,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:19,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:19,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:20,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:20,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:11:20,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:11:21,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:22,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:23,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:24,285 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:24,389 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:25,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:26,389 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:27,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:27,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:11:27,993 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:11:27,993 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:11:27,994 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:11:28,301 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:11:28,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:28,642 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:11:29,303 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:11:29,407 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:29,644 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:30,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:31,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:32,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:33,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:34,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:35,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:35,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:35,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:11:35,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:11:36,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:37,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:38,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:39,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:40,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:41,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:41,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:42,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:43,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:44,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:45,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:46,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:46,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:47,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:48,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:49,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:50,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:50,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:11:50,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:11:51,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:51,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:52,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:53,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:54,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:55,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:56,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:57,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:57,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:11:58,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:11:58,651 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:11:59,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:11:59,404 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:11:59,404 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:11:59,405 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:11:59,824 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:00,331 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:12:00,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:01,332 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:12:01,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:02,446 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:02,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:03,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:04,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:05,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:05,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:12:05,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:12:06,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:07,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:07,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:08,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:09,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:10,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:11,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:12,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:13,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:13,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:14,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:15,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:16,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:17,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:18,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:19,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:19,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:20,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:20,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:12:20,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:12:21,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:22,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:23,561 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:24,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:24,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:25,564 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:26,564 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:27,564 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:28,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:12:28,425 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:12:28,425 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:12:28,426 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:12:28,655 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:12:28,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:29,360 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:12:29,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:29,658 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:30,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:31,361 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:12:31,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:32,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:33,585 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:34,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:35,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:35,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:35,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:12:35,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:12:36,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:37,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:38,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:39,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:40,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:41,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:41,586 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:42,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:43,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:44,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:45,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:46,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:46,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:47,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:48,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:49,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:50,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:50,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:12:50,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:12:51,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:51,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:52,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:53,587 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:54,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:55,591 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:56,593 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:57,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:12:57,593 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:58,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:12:58,657 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:12:59,593 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:00,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:13:00,209 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:13:00,210 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:13:00,215 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:13:00,388 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:13:00,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:01,389 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:13:01,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:02,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:03,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:03,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:04,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:05,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:05,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:13:05,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:13:06,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:07,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:08,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:08,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:09,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:10,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:11,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:12,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:13,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:13,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:14,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:15,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:16,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:17,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:18,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:19,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:19,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:20,620 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:20,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:13:20,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:13:21,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:22,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:23,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:24,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:24,632 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:25,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:26,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:27,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:13:27,380 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:13:27,381 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:13:27,382 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:13:27,418 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:13:27,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:28,662 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:13:28,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:29,419 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:13:29,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:29,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:30,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:31,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:32,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:33,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:34,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:35,421 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:35,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:13:35,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:13:35,795 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:36,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:37,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:38,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:39,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:40,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:41,421 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:41,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:42,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:43,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:44,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:45,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:46,421 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:46,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:47,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:48,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:49,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:50,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:13:50,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:13:50,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:51,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:51,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:52,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:53,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:54,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:55,949 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:56,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:57,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:13:57,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:58,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:13:58,301 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:13:58,301 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:13:58,303 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:13:58,448 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:13:58,665 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:13:58,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:13:59,449 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:13:59,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:00,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:01,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:02,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:02,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:03,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:04,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:05,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:14:05,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:14:05,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:06,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:07,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:07,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:08,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:09,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:11,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:12,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:13,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:13,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:14,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:15,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:16,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:17,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:18,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:19,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:19,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:20,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:20,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:14:20,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:14:21,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:22,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:23,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:24,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:24,448 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:25,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:26,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:27,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:28,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:28,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:14:28,561 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:14:28,561 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:14:28,563 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:14:28,669 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:14:29,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:29,476 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:14:29,672 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:30,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:31,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:31,477 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:14:32,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:33,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:34,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:35,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:35,449 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:35,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:14:35,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:14:36,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:37,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:38,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:39,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:40,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:41,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:41,449 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:42,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:43,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:44,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:45,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:46,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:46,449 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:47,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:48,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:49,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:50,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:50,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:14:50,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:14:51,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:51,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:52,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:53,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:54,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:55,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:56,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:57,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:57,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:14:58,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:58,675 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:14:59,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:14:59,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:14:59,623 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:14:59,623 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:14:59,624 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:15:00,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:00,509 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:15:01,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:01,510 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:15:02,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:02,663 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:03,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:04,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:05,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:05,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:15:05,756 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:15:06,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:07,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:08,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:08,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:09,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:10,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:11,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:12,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:13,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:13,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:14,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:15,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:16,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:17,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:18,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:18,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:19,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:20,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:20,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:15:20,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:15:21,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:22,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:23,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:24,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:24,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:25,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:26,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:27,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:28,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:28,684 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:15:29,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:29,685 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:30,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:31,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:31,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:15:31,235 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:15:31,236 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:15:31,237 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:15:31,534 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:15:32,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:33,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:33,536 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:15:34,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:35,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:35,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:35,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:15:35,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:15:36,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:37,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:38,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:39,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:40,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:41,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:41,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:42,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:43,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:44,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:45,084 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:46,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:46,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:47,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:48,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:49,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:50,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:50,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:15:50,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:15:51,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:51,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:52,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:53,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:54,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:55,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:56,089 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:57,102 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:57,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:15:58,088 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:15:58,686 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:15:59,089 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:00,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:01,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:01,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:16:01,388 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:16:01,389 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:16:01,394 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:16:01,560 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:16:02,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:03,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:03,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:03,562 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:16:04,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:05,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:05,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:16:05,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:16:06,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:07,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:08,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:08,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:09,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:10,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:11,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:12,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:13,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:13,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:14,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:15,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:16,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:17,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:18,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:18,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:19,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:20,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:20,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:16:20,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:16:21,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:22,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:23,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:24,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:24,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:25,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:26,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:27,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:28,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:28,690 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:16:29,180 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:29,691 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:30,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:31,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:32,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:32,314 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:16:32,315 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:16:32,316 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:16:32,319 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:16:32,592 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:16:33,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:33,593 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:16:34,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:35,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:35,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:35,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:16:35,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:16:36,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:37,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:38,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:39,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:40,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:41,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:41,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:42,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:43,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:44,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:45,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:46,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:46,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:47,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:48,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:49,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:50,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:50,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:16:50,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:16:51,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:51,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:52,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:53,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:54,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:55,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:56,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:57,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:57,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:16:58,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:16:58,694 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:16:59,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:00,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:01,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:02,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:02,696 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:02,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:17:02,972 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:17:02,972 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:17:02,975 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:17:03,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:03,628 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:17:04,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:05,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:05,630 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:17:05,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:17:05,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:17:06,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:07,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:07,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:08,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:09,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:10,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:11,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:12,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:13,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:13,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:14,364 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:15,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:16,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:17,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:18,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:18,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:19,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:20,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:20,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:17:20,757 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:17:21,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:22,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:23,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:24,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:24,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:25,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:26,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:27,369 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:28,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:28,695 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:17:29,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:29,609 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:17:29,611 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:17:29,612 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:17:29,612 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:29,613 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:17:29,659 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:17:30,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:31,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:31,661 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:17:32,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:33,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:34,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:35,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:35,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:35,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:17:35,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:17:36,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:37,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:38,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:39,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:40,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:41,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:41,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:42,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:43,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:44,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:45,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:46,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:46,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:47,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:48,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:49,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:50,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:50,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:17:50,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:17:51,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:51,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:52,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:53,373 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:54,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:55,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:56,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:57,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:57,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:17:58,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:58,703 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:17:59,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:17:59,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:17:59,835 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:17:59,835 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:17:59,836 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:18:00,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:00,693 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:18:01,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:01,694 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:18:02,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:02,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:03,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:04,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:05,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:05,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:18:05,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:18:06,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:07,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:08,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:08,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:09,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:10,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:11,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:12,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:13,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:13,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:14,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:15,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:16,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:17,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:18,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:18,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:19,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:20,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:20,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:18:20,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:18:21,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:22,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:23,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:24,384 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:24,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:25,388 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:26,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:27,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:28,387 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:28,708 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:18:29,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:18:29,251 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:18:29,251 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:18:29,252 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:18:29,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:29,716 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:18:30,291 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:30,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:31,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:31,718 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:18:32,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:33,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:34,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:35,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:35,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:35,757 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:18:35,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:18:36,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:37,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:38,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:39,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:40,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:41,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:41,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:42,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:43,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:44,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:45,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:46,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:46,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:47,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:48,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:49,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:50,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:50,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:18:50,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:18:51,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:51,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:52,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:53,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:54,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:55,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:56,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:57,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:57,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:18:58,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:58,715 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:18:59,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:18:59,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:18:59,585 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:18:59,586 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:18:59,587 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:18:59,750 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:19:00,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:01,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:01,751 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:19:02,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:02,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:03,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:04,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:05,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:05,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:19:05,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:19:06,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:07,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:07,838 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:08,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:09,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:10,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:11,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:12,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:13,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:13,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:14,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:15,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:16,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:17,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:18,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:18,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:19,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:20,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:20,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:19:20,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:19:21,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:22,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:23,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:23,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:24,421 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:25,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:26,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:27,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:28,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:28,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:28,715 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:19:28,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:19:28,721 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:19:28,722 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:19:28,723 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:19:28,774 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:19:29,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:29,775 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:19:30,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:31,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:32,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:33,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:34,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:34,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:35,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:35,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:19:35,758 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:19:36,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:37,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:38,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:39,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:40,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:40,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:41,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:42,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:43,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:44,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:45,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:45,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:46,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:47,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:48,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:49,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:50,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:50,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:19:50,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:19:50,919 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:51,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:52,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:53,500 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:54,494 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:55,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:56,495 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:56,500 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:19:57,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:58,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:58,719 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:19:59,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:19:59,021 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:19:59,021 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:19:59,023 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:19:59,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:19:59,804 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:19:59,805 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:20:00,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:01,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:02,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:02,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:03,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:04,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:05,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:05,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:20:05,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:20:06,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:07,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:07,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:08,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:09,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:10,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:11,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:12,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:13,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:13,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:14,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:15,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:16,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:17,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:18,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:19,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:19,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:20,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:20:20,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:20:20,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:21,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:22,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:23,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:24,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:24,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:25,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:26,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:27,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:28,089 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:20:28,090 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:20:28,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:20:28,092 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:20:28,723 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:20:28,831 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:20:28,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:29,726 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:29,832 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:20:29,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:30,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:31,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:32,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:33,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:34,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:35,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:35,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:20:35,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:20:35,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:36,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:37,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:38,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:39,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:40,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:41,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:41,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:42,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:43,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:44,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:45,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:46,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:47,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:47,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:48,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:49,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:50,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:20:50,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:20:50,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:51,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:52,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:52,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:53,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:54,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:55,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:56,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:57,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:58,384 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:20:58,385 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:20:58,385 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:20:58,385 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:20:58,386 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:20:58,727 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:20:58,869 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:20:58,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:20:59,870 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:20:59,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:00,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:01,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:02,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:03,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:03,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:04,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:05,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:21:05,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:21:05,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:06,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:07,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:08,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:09,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:09,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:10,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:12,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:13,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:14,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:14,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:15,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:16,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:17,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:18,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:19,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:19,531 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:20,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:20,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:21:20,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:21:21,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:22,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:23,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:24,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:24,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:25,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:26,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:27,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:28,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:28,730 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:21:29,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:29,450 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:21:29,451 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:21:29,452 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:21:29,454 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:21:29,901 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:21:30,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:30,491 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:31,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:31,902 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:21:32,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:33,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:34,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:35,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:35,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:35,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:21:35,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:21:36,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:37,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:38,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:39,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:40,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:40,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:41,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:42,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:43,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:44,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:45,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:45,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:46,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:47,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:48,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:49,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:50,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:50,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:21:50,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:21:50,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:51,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:52,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:53,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:54,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:55,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:56,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:56,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:21:57,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:58,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:21:58,736 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:21:59,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:00,127 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:01,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:01,749 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:02,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:02,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:22:02,911 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:22:02,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:22:02,913 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:22:02,941 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:22:03,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:03,943 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:22:04,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:05,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:05,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:22:05,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:22:06,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:06,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:07,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:08,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:09,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:10,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:11,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:12,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:12,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:13,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:14,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:15,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:16,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:17,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:18,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:18,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:19,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:20,311 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:20,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:22:20,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:22:21,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:22,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:23,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:24,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:24,541 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:25,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:26,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:27,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:28,322 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:28,737 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:22:29,315 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:29,741 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:30,315 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:31,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:32,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:33,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:33,511 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:22:33,513 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:22:33,513 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:22:33,517 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:22:33,972 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:22:33,973 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:22:34,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:34,942 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:35,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:35,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:22:35,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:22:36,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:37,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:38,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:39,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:40,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:40,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:41,316 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:42,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:43,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:44,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:45,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:46,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:46,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:47,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:48,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:49,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:50,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:50,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:22:50,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:22:51,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:51,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:52,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:53,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:54,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:55,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:56,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:57,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:57,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:22:58,328 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:22:58,738 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:22:59,322 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:00,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:01,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:02,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:02,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:03,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:23:03,302 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:23:03,302 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:23:03,304 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:23:03,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:03,997 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:23:03,997 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:23:04,702 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:05,702 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:05,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:23:05,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:23:06,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:07,702 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:07,868 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:08,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:09,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:10,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:11,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:12,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:13,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:13,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:14,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:15,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:16,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:17,703 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:18,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:19,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:19,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:20,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:20,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:23:20,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:23:21,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:22,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:23,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:24,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:25,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:25,705 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:26,704 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:27,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:28,711 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:28,744 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:23:29,714 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:30,709 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:30,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:31,709 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:32,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:32,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:23:32,936 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:23:32,936 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:23:32,938 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:23:33,028 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:23:33,709 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:34,029 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:23:34,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:35,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:35,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:23:35,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:23:35,916 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:36,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:37,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:38,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:39,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:40,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:41,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:41,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:42,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:43,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:44,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:45,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:46,706 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:46,711 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:47,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:48,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:49,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:50,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:23:50,761 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:23:50,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:51,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:51,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:52,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:53,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:54,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:55,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:56,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:57,707 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:23:57,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:58,753 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:23:58,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:23:59,779 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:00,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:01,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:02,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:02,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:03,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:03,941 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:24:03,943 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:24:03,944 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:24:03,949 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:24:04,053 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:24:04,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:05,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:24:05,761 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:24:05,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:06,055 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:24:06,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:07,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:07,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:08,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:09,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:10,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:11,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:12,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:13,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:13,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:14,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:15,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:16,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:17,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:18,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:18,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:19,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:20,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:24:20,761 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:24:20,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:21,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:22,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:23,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:24,710 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:24,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:25,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:26,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:27,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:28,752 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:24:28,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:29,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:29,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:30,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:31,788 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:32,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:33,788 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:34,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:35,723 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:35,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:24:35,770 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:24:35,812 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:36,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:36,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:24:36,819 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:24:36,819 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:24:36,824 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:24:37,081 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:24:37,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:38,082 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:24:38,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:39,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:40,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:41,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:41,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:42,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:43,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:44,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:45,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:46,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:47,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:47,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:48,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:49,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:50,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:24:50,767 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:24:50,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:51,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:52,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:52,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:53,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:54,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:55,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:56,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:57,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:58,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:24:58,756 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:24:58,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:24:59,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:00,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:01,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:02,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:03,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:04,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:04,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:05,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:25:05,767 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:25:05,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:06,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:07,653 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:25:07,655 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:25:07,656 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:25:07,661 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:25:08,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:08,119 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:25:08,120 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:25:09,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:10,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:10,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:11,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:12,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:13,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:14,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:15,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:15,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:16,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:17,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:18,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:19,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:20,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:20,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:25:20,768 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:25:21,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:21,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:22,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:23,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:24,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:25,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:26,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:26,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:27,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:28,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:28,760 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:25:29,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:30,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:31,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:31,763 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:32,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:33,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:34,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:35,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:35,768 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:25:35,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:25:36,073 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:36,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:37,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:38,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:38,462 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:25:38,464 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:25:38,464 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:25:38,467 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:25:39,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:39,147 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:25:40,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:40,148 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:25:41,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:42,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:42,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:43,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:44,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:45,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:46,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:47,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:48,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:48,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:49,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:50,071 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:50,768 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:25:50,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:25:51,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:52,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:53,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:53,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:25:54,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:55,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:56,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:57,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:58,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:58,765 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:25:59,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:25:59,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:00,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:01,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:02,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:03,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:04,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:04,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:05,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:05,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:26:05,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:26:06,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:07,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:08,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:09,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:09,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:26:09,148 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:26:09,148 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:26:09,153 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:26:09,173 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:26:10,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:10,174 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:26:10,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:11,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:12,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:13,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:14,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:15,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:15,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:16,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:17,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:18,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:19,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:20,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:20,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:26:20,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:26:20,932 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:21,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:22,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:23,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:24,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:25,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:26,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:26,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:27,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:28,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:28,769 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:26:29,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:30,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:31,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:32,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:32,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:33,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:34,150 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:35,150 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:35,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:26:35,777 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:26:36,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:37,150 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:37,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:38,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:39,150 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:39,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:26:39,557 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:26:39,557 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:26:39,559 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:26:40,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:40,199 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:26:40,199 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:26:41,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:42,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:43,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:43,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:44,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:45,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:46,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:47,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:48,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:48,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:49,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:50,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:50,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:26:50,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:26:51,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:52,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:53,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:53,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:26:54,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:55,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:56,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:57,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:58,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:58,773 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:26:59,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:26:59,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:00,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:01,157 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:02,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:03,161 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:04,161 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:04,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:05,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:27:05,124 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:27:05,124 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:27:05,125 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:27:05,227 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:27:05,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:05,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:27:05,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:27:06,228 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:27:06,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:07,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:08,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:09,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:10,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:10,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:11,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:12,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:13,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:14,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:15,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:16,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:16,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:17,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:18,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:19,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:20,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:20,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:27:20,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:27:21,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:21,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:22,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:23,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:24,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:25,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:26,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:27,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:27,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:28,522 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:28,777 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:27:29,522 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:30,532 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:31,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:32,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:32,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:33,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:34,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:35,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:35,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:27:35,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:27:35,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:27:35,908 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:27:35,909 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:27:35,910 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:27:36,258 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:27:36,258 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:27:36,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:37,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:37,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:38,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:39,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:40,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:41,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:42,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:43,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:43,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:44,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:45,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:46,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:47,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:48,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:48,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:49,527 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:50,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:50,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:27:50,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:27:51,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:52,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:53,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:54,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:54,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:27:55,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:56,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:57,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:58,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:58,781 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:27:59,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:27:59,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:00,534 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:01,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:02,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:03,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:04,533 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:05,325 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:28:05,326 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:28:05,326 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:28:05,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:05,327 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:28:05,721 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:05,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:28:05,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:28:06,303 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:28:06,304 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:28:06,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:07,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:08,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:09,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:10,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:10,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:11,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:12,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:13,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:14,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:15,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:15,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:16,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:17,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:18,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:19,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:20,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:20,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:28:20,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:28:20,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:21,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:22,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:23,716 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:24,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:25,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:26,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:26,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:27,717 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:28,719 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:28,790 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:28:29,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:30,726 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:31,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:31,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:32,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:33,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:28:33,469 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:28:33,469 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:28:33,471 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:28:33,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:34,330 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:28:34,331 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:28:34,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:35,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:28:35,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:28:35,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:36,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:37,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:37,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:38,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:39,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:40,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:41,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:42,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:42,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:43,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:44,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:45,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:46,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:47,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:47,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:48,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:49,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:50,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:28:50,771 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:28:50,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:51,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:52,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:52,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:53,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:54,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:55,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:56,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:57,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:58,791 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:28:58,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:28:58,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:28:59,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:00,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:01,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:02,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:03,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:03,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:04,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:04,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:29:04,897 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:29:04,897 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:29:04,902 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:29:05,358 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:29:05,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:29:05,772 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:29:05,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:06,359 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:29:06,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:07,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:08,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:08,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:09,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:10,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:11,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:12,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:13,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:14,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:14,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:15,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:16,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:17,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:18,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:19,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:19,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:20,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:29:20,772 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:29:20,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:21,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:22,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:23,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:24,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:25,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:25,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:26,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:27,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:28,795 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:29:28,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:29,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:30,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:30,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:31,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:32,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:33,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:34,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:35,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:29:35,416 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:29:35,417 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:29:35,422 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:29:35,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:29:35,879 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:29:35,949 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:36,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:36,388 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:29:36,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:37,385 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:29:37,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:38,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:39,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:40,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:41,914 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:41,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:42,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:43,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:44,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:45,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:46,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:47,914 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:47,948 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:48,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:49,950 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:50,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:29:50,878 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:29:50,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:51,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:52,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:52,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:53,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:54,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:55,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:56,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:57,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:58,798 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:29:58,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:29:58,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:29:59,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:00,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:01,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:02,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:03,956 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:03,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:04,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:05,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:30:05,889 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:30:05,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:06,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:07,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:30:07,407 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:30:07,408 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:30:07,413 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:30:07,416 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:30:07,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:08,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:09,419 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:30:09,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:09,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:10,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:11,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:12,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:13,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:14,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:15,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:15,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:16,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:17,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:18,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:19,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:20,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:30:20,881 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:30:20,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:21,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:21,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:22,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:23,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:24,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:25,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:26,952 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:26,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:27,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:28,802 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:30:28,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:29,960 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:30,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:31,957 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:31,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:32,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:33,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:34,969 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:35,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:30:35,882 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:30:35,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:36,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:30:36,516 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:30:36,516 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:30:36,517 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:30:36,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:37,447 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:30:37,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:37,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:38,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:39,449 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:30:39,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:40,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:41,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:42,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:42,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:43,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:44,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:45,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:46,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:47,955 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:47,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:49,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:50,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:50,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:30:50,882 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:30:51,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:52,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:53,024 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:53,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:54,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:55,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:56,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:57,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:58,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:30:58,807 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:30:58,809 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:30:59,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:00,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:01,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:02,066 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:03,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:03,979 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:04,066 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:05,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:05,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:31:05,885 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:31:06,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:06,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:31:06,858 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:31:06,858 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:31:06,860 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:31:07,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:07,482 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:31:08,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:08,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:09,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:09,484 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:31:10,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:11,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:12,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:13,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:13,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:14,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:15,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:16,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:17,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:18,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:19,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:19,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:20,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:20,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:31:20,885 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:31:21,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:22,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:23,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:24,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:24,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:25,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:26,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:27,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:28,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:28,811 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:31:29,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:30,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:30,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:31,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:32,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:33,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:34,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:35,269 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:35,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:31:35,885 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:31:36,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:36,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:36,973 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:31:36,975 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:31:36,975 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:31:36,977 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:31:37,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:37,510 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:31:38,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:39,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:39,512 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:31:40,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:41,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:41,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:42,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:43,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:44,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:45,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:46,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:47,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:47,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:48,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:49,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:50,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:50,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:31:50,886 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:31:51,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:52,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:53,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:53,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:54,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:55,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:56,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:57,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:58,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:31:58,815 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:31:58,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:31:59,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:00,380 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:01,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:02,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:03,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:03,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:04,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:05,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:05,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:32:05,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:32:06,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:06,915 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:32:06,917 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:32:06,917 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:32:06,919 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:32:07,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:07,543 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:32:08,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:09,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:09,544 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:32:09,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:10,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:11,392 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:12,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:13,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:14,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:15,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:15,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:16,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:17,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:18,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:19,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:20,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:20,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:32:20,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:32:21,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:21,393 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:22,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:23,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:24,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:25,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:26,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:26,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:27,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:28,394 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:28,826 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:32:29,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:30,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:31,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:32,008 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:32,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:33,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:34,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:34,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:32:34,405 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:32:34,406 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:32:34,408 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:32:34,573 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:32:35,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:35,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:32:35,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:32:36,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:37,024 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:37,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:37,575 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:32:38,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:39,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:40,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:41,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:42,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:42,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:43,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:44,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:45,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:46,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:47,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:48,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:48,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:49,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:50,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:50,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:32:50,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:32:51,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:52,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:53,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:54,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:54,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:32:55,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:56,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:57,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:58,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:58,821 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:32:59,407 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:32:59,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:00,407 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:01,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:02,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:03,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:03,671 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:33:03,672 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:33:03,672 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:33:03,674 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:33:04,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:04,603 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:33:05,076 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:05,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:05,604 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:33:05,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:33:05,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:33:06,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:07,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:08,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:09,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:10,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:11,074 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:11,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:12,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:13,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:14,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:15,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:16,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:17,074 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:17,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:18,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:19,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:20,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:20,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:33:20,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:33:21,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:22,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:23,013 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:23,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:24,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:25,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:26,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:27,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:28,083 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:28,425 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:28,827 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:33:29,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:30,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:31,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:32,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:33,089 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:33,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:33,571 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:33:33,572 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:33:33,572 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:33:33,574 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:33:33,629 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:33:34,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:35,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:35,631 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:33:35,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:33:35,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:33:36,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:37,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:38,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:38,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:39,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:40,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:41,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:42,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:43,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:44,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:44,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:45,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:46,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:47,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:48,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:49,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:49,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:50,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:50,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:33:50,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:33:51,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:52,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:53,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:54,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:55,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:33:55,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:56,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:57,425 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:58,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:33:58,832 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:33:59,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:00,079 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:00,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:01,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:02,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:34:02,373 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:34:02,373 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:34:02,375 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:34:02,663 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:34:02,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:03,664 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:34:03,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:04,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:05,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:05,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:34:05,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:34:06,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:06,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:07,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:08,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:09,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:10,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:11,077 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:11,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:12,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:13,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:14,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:15,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:16,078 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:16,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:17,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:18,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:19,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:20,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:20,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:34:20,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:34:21,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:21,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:22,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:23,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:24,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:25,778 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:26,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:27,082 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:27,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:28,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:28,844 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:34:29,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:30,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:31,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:31,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:34:31,787 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:34:31,788 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:34:31,789 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:34:32,689 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:34:32,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:32,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:33,690 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:34:33,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:34,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:35,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:35,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:34:35,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:34:36,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:37,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:38,020 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:38,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:39,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:40,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:41,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:42,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:43,082 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:43,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:44,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:45,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:46,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:47,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:48,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:49,081 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:49,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:50,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:50,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:34:50,891 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:34:51,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:52,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:53,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:54,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:55,081 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:34:55,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:56,788 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:57,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:58,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:34:58,846 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:34:59,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:00,088 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:00,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:01,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:02,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:02,943 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:35:02,944 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:35:02,945 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:35:02,946 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:35:03,722 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:35:03,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:04,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:05,724 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:35:05,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:05,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:35:05,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:35:06,001 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:06,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:07,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:08,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:09,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:10,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:11,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:11,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:12,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:13,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:14,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:15,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:16,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:17,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:17,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:18,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:19,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:20,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:20,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:35:20,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:35:21,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:22,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:23,010 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:23,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:24,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:25,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:26,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:27,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:28,188 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:28,845 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:35:28,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:29,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:30,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:31,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:32,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:33,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:34,086 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:34,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:35,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:35:35,054 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:35:35,055 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:35:35,060 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:35:35,753 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:35:35,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:35,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:35:35,898 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:35:36,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:37,755 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:35:37,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:38,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:39,087 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:39,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:40,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:41,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:42,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:43,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:44,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:45,087 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:45,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:46,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:47,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:48,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:49,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:50,087 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:50,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:50,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:35:50,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:35:51,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:52,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:53,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:54,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:55,087 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:35:55,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:56,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:57,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:58,849 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:35:58,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:35:59,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:00,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:00,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:01,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:02,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:03,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:04,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:04,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:36:04,991 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:36:04,991 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:36:04,993 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:36:05,780 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:36:05,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:05,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:36:05,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:36:06,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:06,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:07,781 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:36:07,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:08,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:09,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:10,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:11,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:11,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:12,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:13,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:14,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:15,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:16,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:17,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:17,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:18,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:19,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:20,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:20,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:36:20,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:36:21,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:22,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:23,002 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:23,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:24,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:25,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:26,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:27,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:28,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:28,853 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:36:28,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:29,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:30,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:31,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:32,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:33,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:33,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:34,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:35,431 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:36:35,433 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:36:35,433 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:36:35,435 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:36:35,807 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:36:35,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:35,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:36:35,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:36:36,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:37,808 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:36:37,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:38,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:39,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:39,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:40,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:41,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:42,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:43,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:44,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:44,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:45,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:46,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:47,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:48,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:49,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:49,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:50,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:50,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:36:50,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:36:51,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:52,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:53,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:54,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:55,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:36:55,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:56,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:57,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:58,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:36:58,870 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:36:59,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:00,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:00,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:01,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:02,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:03,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:04,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:37:04,254 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:37:04,254 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:37:04,256 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:37:04,834 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:37:04,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:05,294 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:05,835 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:37:05,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:05,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:37:05,892 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:37:06,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:07,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:08,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:09,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:10,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:11,103 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:11,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:12,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:13,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:14,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:15,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:16,103 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:16,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:17,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:18,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:19,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:20,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:20,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:37:20,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:37:21,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:22,020 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:22,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:23,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:24,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:25,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:26,879 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:27,103 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:27,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:28,869 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:37:28,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:29,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:30,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:31,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:32,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:32,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:33,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:33,986 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:37:33,988 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:37:33,988 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:37:33,989 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:37:34,858 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:37:34,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:35,859 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:37:35,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:35,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:37:35,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:37:36,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:37,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:38,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:38,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:39,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:40,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:41,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:42,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:43,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:43,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:44,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:45,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:46,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:47,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:48,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:49,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:49,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:50,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:50,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:37:50,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:37:51,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:52,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:53,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:54,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:54,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:55,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:56,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:57,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:58,877 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:37:58,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:37:59,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:37:59,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:00,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:01,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:02,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:03,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:04,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:38:04,142 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:38:04,143 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:38:04,144 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:38:04,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:04,890 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:38:04,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:05,891 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:38:05,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:38:05,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:38:05,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:06,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:07,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:08,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:09,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:10,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:10,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:11,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:12,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:13,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:14,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:15,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:16,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:16,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:17,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:18,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:19,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:20,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:38:20,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:38:20,933 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:21,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:21,984 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:22,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:23,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:24,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:25,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:26,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:27,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:27,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:28,867 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:38:28,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:29,843 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:38:29,845 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:38:29,846 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:38:29,848 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:38:29,914 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:38:30,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:30,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:31,916 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:38:31,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:32,886 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:32,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:33,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:34,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:35,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:38:35,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:38:35,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:36,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:37,989 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:38,032 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:38,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:39,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:40,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:41,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:42,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:43,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:43,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:44,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:45,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:46,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:47,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:48,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:48,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:49,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:50,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:38:50,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:38:50,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:51,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:52,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:54,002 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:54,017 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:54,998 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:55,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:56,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:57,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:58,870 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:38:59,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:38:59,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:38:59,086 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:38:59,086 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:38:59,087 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:38:59,088 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:38:59,945 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:38:59,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:00,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:01,947 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:39:01,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:03,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:04,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:04,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:05,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:05,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:39:05,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:39:06,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:07,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:08,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:09,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:10,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:10,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:11,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:12,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:13,032 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:14,032 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:15,032 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:15,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:16,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:17,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:18,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:19,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:20,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:20,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:39:20,893 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:39:20,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:21,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:22,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:23,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:24,044 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:25,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:26,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:26,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:27,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:28,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:28,874 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:39:29,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:29,631 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:39:29,633 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:39:29,633 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:39:29,635 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:39:29,972 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:39:30,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:31,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:31,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:31,973 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:39:32,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:33,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:34,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:35,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:35,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:39:35,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:39:36,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:37,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:37,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:38,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:39,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:40,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:41,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:42,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:42,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:43,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:44,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:45,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:46,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:47,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:48,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:48,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:49,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:50,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:50,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:39:50,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:39:51,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:52,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:53,043 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:54,041 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:54,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:55,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:56,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:57,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:58,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:58,880 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:39:59,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:39:59,240 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:39:59,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:39:59,481 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:39:59,481 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:39:59,483 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:39:59,999 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:40:00,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:01,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:02,002 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:40:02,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:03,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:04,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:05,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:05,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:05,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:40:05,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:40:06,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:07,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:08,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:09,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:10,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:11,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:11,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:12,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:13,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:14,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:15,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:16,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:16,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:17,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:18,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:19,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:20,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:20,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:40:20,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:40:21,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:22,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:22,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:23,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:24,054 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:25,055 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:26,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:27,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:27,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:28,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:28,882 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:40:29,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:30,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:30,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:40:30,948 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:40:30,948 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:40:30,950 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:40:31,029 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:40:31,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:32,029 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:40:32,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:32,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:33,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:34,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:35,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:35,893 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:40:35,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:40:36,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:37,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:37,995 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:38,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:39,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:40,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:41,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:42,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:43,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:43,295 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:44,201 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:45,201 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:46,201 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:47,201 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:48,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:48,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:49,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:50,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:50,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:40:50,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:40:51,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:52,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:53,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:54,061 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:40:54,202 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:55,213 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:56,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:57,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:58,213 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:58,890 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:40:59,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:40:59,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:00,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:00,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:41:00,778 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:41:00,778 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:41:00,779 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:41:01,062 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:41:01,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:02,063 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:41:02,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:03,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:04,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:05,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:05,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:05,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:41:05,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:41:06,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:07,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:08,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:09,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:10,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:11,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:11,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:12,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:13,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:14,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:15,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:16,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:16,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:17,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:18,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:19,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:20,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:20,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:41:20,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:41:21,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:22,014 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:22,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:23,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:24,211 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:25,221 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:26,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:27,216 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:27,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:28,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:28,892 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:41:29,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:30,219 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:30,832 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:41:30,834 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:41:30,834 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:41:30,835 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:41:31,095 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:41:31,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:32,096 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:41:32,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:32,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:33,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:34,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:35,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:35,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:41:35,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:41:36,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:37,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:38,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:38,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:39,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:40,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:41,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:42,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:43,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:43,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:44,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:45,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:46,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:47,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:48,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:48,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:49,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:50,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:50,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:41:50,894 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:41:51,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:52,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:53,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:53,977 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:41:54,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:55,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:56,235 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:57,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:58,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:58,896 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:41:59,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:41:59,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:00,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:01,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:02,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:02,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:42:02,406 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:42:02,406 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:42:02,409 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:42:03,124 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:42:03,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:04,125 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:42:04,241 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:05,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:05,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:05,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:42:05,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:42:06,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:07,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:08,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:09,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:10,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:11,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:11,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:12,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:13,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:14,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:15,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:16,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:16,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:17,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:18,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:19,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:20,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:20,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:42:20,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:42:21,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:22,031 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:22,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:23,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:24,243 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:25,244 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:26,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:27,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:27,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:28,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:28,901 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:42:29,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:30,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:31,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:31,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:42:31,404 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:42:31,404 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:42:31,406 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:42:32,148 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:42:32,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:32,443 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:33,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:34,150 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:42:34,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:35,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:35,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:42:35,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:42:36,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:37,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:38,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:38,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:39,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:40,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:41,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:42,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:43,251 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:43,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:44,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:45,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:46,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:47,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:48,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:49,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:49,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:50,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:50,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:42:50,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:42:51,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:52,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:53,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:54,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:55,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:55,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:42:56,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:57,253 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:58,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:42:58,901 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:42:59,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:00,257 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:00,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:01,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:02,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:02,305 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:43:02,306 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:43:02,307 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:43:02,308 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:43:03,176 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:43:03,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:04,177 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:43:04,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:05,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:05,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:43:05,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:43:06,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:06,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:07,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:08,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:09,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:10,258 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:11,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:11,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:12,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:13,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:14,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:15,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:16,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:16,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:17,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:18,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:19,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:20,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:20,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:43:20,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:43:21,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:21,985 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:22,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:23,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:24,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:25,260 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:26,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:27,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:27,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:28,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:28,908 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:43:29,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:30,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:31,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:31,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:43:31,345 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:43:31,346 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:43:31,348 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:43:32,206 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:43:32,206 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:43:32,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:32,386 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:33,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:34,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:35,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:35,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:43:35,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:43:36,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:37,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:37,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:38,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:39,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:40,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:41,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:42,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:43,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:43,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:44,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:45,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:46,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:47,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:48,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:48,270 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:49,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:50,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:50,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:43:50,895 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:43:51,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:52,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:53,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:53,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:54,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:55,267 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:56,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:57,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:58,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:43:58,912 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:43:59,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:43:59,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:00,278 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:01,279 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:01,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:44:01,313 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:44:01,313 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:44:01,315 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:44:02,242 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:44:02,242 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:44:02,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:03,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:04,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:05,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:05,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:05,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:44:05,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:44:06,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:07,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:08,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:09,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:10,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:11,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:11,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:12,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:13,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:14,281 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:15,407 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:16,274 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:16,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:17,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:18,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:19,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:20,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:20,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:44:20,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:44:21,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:21,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:22,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:23,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:24,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:25,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:26,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:27,279 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:27,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:28,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:28,914 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:44:29,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:30,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:31,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:32,103 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:44:32,104 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:44:32,104 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:44:32,107 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:44:32,273 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:44:32,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:33,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:33,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:34,276 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:44:34,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:35,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:35,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:44:35,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:44:36,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:37,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:38,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:38,977 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:39,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:40,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:41,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:42,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:43,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:44,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:44,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:45,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:46,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:47,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:48,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:49,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:49,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:50,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:50,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:44:50,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:44:51,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:52,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:53,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:54,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:55,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:44:55,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:56,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:57,506 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:58,516 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:44:58,921 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:44:59,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:00,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:00,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:01,515 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:02,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:03,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:45:03,403 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:45:03,403 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:45:03,405 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:45:03,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:04,307 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:45:04,308 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:45:04,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:05,285 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:05,550 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:05,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:45:05,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:45:06,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:07,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:08,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:09,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:10,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:11,279 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:11,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:12,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:13,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:14,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:15,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:16,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:17,279 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:17,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:18,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:19,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:20,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:20,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:45:20,896 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:45:21,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:22,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:23,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:23,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:24,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:25,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:26,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:27,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:28,280 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:28,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:28,925 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:45:29,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:30,549 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:31,552 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:32,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:33,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:33,564 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:34,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:35,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:35,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:45:35,898 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:45:36,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:37,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:45:37,010 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:45:37,010 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:45:37,013 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:45:37,335 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:45:37,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:38,337 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:45:38,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:39,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:39,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:40,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:41,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:42,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:43,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:44,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:44,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:45,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:46,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:47,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:48,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:49,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:49,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:50,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:50,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:45:50,898 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:45:51,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:52,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:53,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:54,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:54,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:55,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:56,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:57,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:58,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:45:58,929 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:45:59,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:45:59,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:00,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:01,574 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:02,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:03,581 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:04,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:04,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:05,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:05,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:46:05,902 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:46:06,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:06,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:46:06,670 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:46:06,671 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:46:06,672 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:46:07,373 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:46:07,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:08,374 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:46:08,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:09,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:10,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:10,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:11,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:12,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:13,582 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:14,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:15,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:16,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:16,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:17,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:18,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:19,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:20,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:20,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:46:20,899 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:46:21,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:22,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:22,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:23,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:24,583 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:25,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:26,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:27,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:27,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:28,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:28,933 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:46:29,584 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:30,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:31,591 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:32,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:32,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:33,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:34,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:35,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:35,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:46:35,899 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:46:35,978 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:46:35,980 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:46:35,980 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:46:35,981 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:46:36,398 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:46:36,399 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:46:36,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:37,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:37,982 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:38,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:39,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:40,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:41,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:42,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:43,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:43,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:44,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:45,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:46,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:47,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:48,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:49,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:49,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:50,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:50,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:46:50,899 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:46:51,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:52,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:53,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:54,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:55,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:46:55,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:56,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:57,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:58,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:46:58,938 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:46:59,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:00,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:01,350 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:01,617 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:02,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:03,611 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:04,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:05,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:05,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:47:05,910 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:47:06,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:07,009 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:07,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:47:07,035 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:47:07,035 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:47:07,037 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:47:07,429 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:47:07,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:08,430 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:47:08,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:09,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:10,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:11,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:12,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:12,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:13,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:14,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:15,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:16,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:17,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:18,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:18,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:19,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:20,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:20,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:47:20,904 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:47:21,614 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:22,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:23,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:24,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:24,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:25,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:26,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:27,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:28,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:28,942 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:47:29,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:29,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:30,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:31,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:32,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:33,615 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:34,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:34,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:35,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:35,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:47:35,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:47:36,616 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:37,619 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:38,623 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:39,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:39,623 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:40,628 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:41,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:42,627 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:43,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:47:43,425 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:47:43,425 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:47:43,426 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:47:43,467 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:47:43,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:44,468 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:47:44,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:45,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:45,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:46,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:47,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:48,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:49,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:50,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:50,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:47:50,904 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:47:51,018 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:51,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:52,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:53,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:54,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:55,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:56,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:47:56,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:57,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:58,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:47:58,946 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:47:59,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:00,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:01,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:01,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:02,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:03,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:04,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:05,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:05,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:48:05,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:48:06,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:07,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:07,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:08,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:09,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:10,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:11,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:12,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:12,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:13,869 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:14,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:15,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:48:15,168 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:48:15,169 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:48:15,171 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:48:15,506 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:48:15,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:16,506 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:48:16,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:17,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:18,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:18,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:19,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:20,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:20,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:48:20,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:48:21,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:22,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:23,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:24,030 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:24,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:25,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:26,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:27,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:28,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:28,951 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:48:29,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:29,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:30,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:31,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:32,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:33,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:34,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:35,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:35,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:35,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:48:35,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:48:36,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:37,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:38,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:39,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:40,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:40,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:41,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:42,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:43,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:44,885 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:45,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:45,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:46,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:47,268 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:48:47,270 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:48:47,270 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:48:47,271 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:48:47,538 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:48:47,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:48,539 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:48:48,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:49,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:50,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:50,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:48:50,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:48:51,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:51,882 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:52,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:53,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:54,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:55,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:56,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:48:56,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:57,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:58,883 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:48:58,954 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:49:00,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:00,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:01,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:01,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:02,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:03,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:04,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:05,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:49:05,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:49:05,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:06,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:06,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:07,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:08,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:09,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:10,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:11,938 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:12,370 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:12,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:13,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:14,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:15,942 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:16,942 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:17,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:17,731 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:49:17,733 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:49:17,733 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:49:17,734 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:49:18,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:18,571 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:49:18,572 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:49:19,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:20,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:20,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:49:20,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:49:21,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:22,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:23,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:23,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:24,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:25,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:26,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:27,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:28,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:28,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:28,958 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:49:29,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:30,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:31,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:32,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:33,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:33,365 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:34,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:35,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:35,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:49:35,905 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:49:36,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:37,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:38,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:39,021 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:39,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:40,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:41,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:42,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:43,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:44,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:44,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:45,162 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:46,161 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:47,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:48,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:49,164 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:49,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:49,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:49:49,410 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:49:49,410 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:49:49,412 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:49:49,603 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:49:50,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:50,604 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:49:50,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:49:50,906 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:49:51,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:52,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:53,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:54,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:55,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:55,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:49:56,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:57,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:58,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:49:58,962 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:49:59,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:00,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:01,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:01,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:02,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:03,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:04,166 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:05,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:05,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:50:05,906 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:50:06,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:07,031 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:07,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:08,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:09,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:10,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:11,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:12,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:12,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:13,167 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:14,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:15,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:16,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:17,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:18,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:18,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:19,168 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:20,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:20,918 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:50:20,919 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:50:21,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:22,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:23,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:24,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:24,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:24,915 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:50:24,916 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:50:24,916 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:50:24,917 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:50:25,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:25,640 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:50:26,312 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:26,641 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:50:27,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:28,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:28,966 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:50:29,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:29,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:30,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:31,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:32,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:33,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:34,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:35,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:35,371 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:35,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:50:35,911 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:50:36,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:37,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:38,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:39,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:40,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:40,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:41,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:42,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:43,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:44,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:45,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:46,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:46,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:47,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:48,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:49,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:50,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:50,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:50:50,911 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:50:51,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:52,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:52,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:53,422 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:54,403 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:55,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:56,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:57,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:57,404 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:50:58,405 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:50:58,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:50:58,558 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:50:58,558 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:50:58,560 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:50:58,684 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:50:58,971 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:50:59,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:00,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:00,686 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:51:01,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:02,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:03,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:03,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:04,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:05,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:05,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:51:05,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:51:06,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:07,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:08,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:09,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:09,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:10,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:11,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:12,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:13,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:14,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:14,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:15,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:16,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:17,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:18,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:19,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:19,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:20,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:20,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:51:20,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:51:21,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:22,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:23,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:24,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:25,409 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:25,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:26,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:27,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:28,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:28,974 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:51:29,217 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:51:29,219 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:51:29,220 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:51:29,221 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:51:29,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:29,718 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:51:30,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:30,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:30,719 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:51:31,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:32,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:33,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:34,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:35,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:35,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:51:35,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:51:35,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:36,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:37,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:38,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:39,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:40,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:41,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:41,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:42,814 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:43,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:44,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:45,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:46,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:47,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:47,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:48,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:49,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:50,810 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:50,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:51:50,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:51:51,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:52,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:53,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:53,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:54,811 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:55,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:56,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:57,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:58,628 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:51:58,821 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:51:58,985 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:51:59,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:00,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:01,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:52:01,477 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:52:01,478 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:52:01,480 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:52:01,755 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:52:01,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:02,756 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:52:02,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:03,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:04,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:04,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:05,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:05,912 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:52:05,912 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:52:06,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:07,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:08,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:09,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:09,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:10,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:11,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:12,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:13,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:14,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:14,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:15,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:16,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:17,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:18,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:19,888 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:20,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:20,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:52:20,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:52:21,032 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:21,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:22,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:23,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:24,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:25,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:25,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:26,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:27,896 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:28,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:28,989 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:52:29,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:30,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:52:30,421 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:52:30,421 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:52:30,423 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:52:30,784 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:52:30,785 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:52:30,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:31,462 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:31,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:32,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:33,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:34,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:35,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:35,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:52:35,991 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:52:36,898 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:37,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:37,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:38,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:39,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:40,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:41,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:42,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:42,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:43,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:44,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:45,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:46,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:47,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:48,622 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:48,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:49,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:50,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:50,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:52:50,991 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:52:51,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:52,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:53,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:54,083 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:54,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:55,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:56,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:57,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:58,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:52:58,992 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:52:59,322 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:52:59,323 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:52:59,323 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:52:59,324 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:52:59,325 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:52:59,814 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:52:59,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:00,815 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:53:00,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:01,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:02,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:03,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:04,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:04,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:05,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:05,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:53:05,991 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:53:06,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:07,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:08,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:09,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:10,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:11,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:12,127 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:13,127 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:14,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:15,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:16,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:16,724 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:17,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:18,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:19,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:20,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:20,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:53:20,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:53:21,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:22,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:22,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:23,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:24,137 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:25,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:26,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:27,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:27,730 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:27,887 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:53:27,889 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:53:27,889 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:53:27,891 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:53:28,266 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:28,848 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:53:28,848 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:53:28,996 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:53:29,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:30,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:31,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:32,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:33,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:33,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:34,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:35,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:35,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:53:35,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:53:36,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:37,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:38,261 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:39,160 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:39,444 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:40,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:41,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:42,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:43,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:44,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:44,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:45,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:46,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:47,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:48,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:49,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:50,352 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:50,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:50,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:53:50,995 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:53:51,354 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:52,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:53,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:54,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:55,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:55,735 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:53:56,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:56,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:53:56,614 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:53:56,614 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:53:56,616 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:53:56,879 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:53:57,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:57,880 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:53:58,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:53:59,000 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:53:59,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:00,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:01,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:01,732 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:02,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:03,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:04,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:05,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:05,991 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:54:05,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:54:06,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:07,088 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:07,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:08,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:09,562 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:10,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:11,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:12,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:12,732 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:13,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:14,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:15,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:16,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:17,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:18,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:18,732 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:19,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:20,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:20,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:54:20,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:54:21,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:22,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:23,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:24,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:24,479 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:25,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:26,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:27,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:27,565 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:54:27,567 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:54:27,567 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:54:27,569 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:54:27,912 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:54:28,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:29,004 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:54:29,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:29,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:29,914 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:54:30,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:31,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:32,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:33,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:34,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:35,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:35,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:35,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:54:35,992 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:54:36,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:37,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:38,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:39,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:40,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:40,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:41,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:42,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:43,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:44,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:45,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:46,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:46,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:47,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:48,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:49,484 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:50,484 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:51,000 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:54:51,002 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:54:51,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:52,085 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:52,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:53,490 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:54,490 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:55,490 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:56,378 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:54:56,379 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:54:56,379 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:54:56,381 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:54:56,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:56,941 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:54:57,419 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:54:57,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:57,942 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:54:58,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:54:59,008 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:54:59,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:00,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:01,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:02,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:02,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:03,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:04,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:05,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:05,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:55:05,996 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:55:06,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:07,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:08,088 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:08,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:09,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:10,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:11,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:12,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:13,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:13,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:14,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:15,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:16,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:17,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:18,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:19,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:19,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:20,777 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:20,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:55:20,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:55:21,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:22,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:23,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:24,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:24,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:25,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:26,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:27,239 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:55:27,241 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:55:27,241 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:55:27,243 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:55:27,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:27,982 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:55:28,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:29,012 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:55:29,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:29,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:29,985 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:55:30,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:31,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:32,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:33,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:34,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:34,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:35,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:35,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:55:35,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:55:36,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:37,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:38,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:39,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:39,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:40,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:41,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:42,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:43,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:44,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:44,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:45,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:46,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:47,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:48,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:49,749 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:49,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:50,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:50,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:55:50,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:55:51,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:52,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:53,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:54,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:55:54,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:55,789 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:56,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:57,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:58,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:55:59,016 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:55:59,718 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:55:59,720 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:55:59,720 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:55:59,722 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:55:59,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:00,022 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:56:00,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:00,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:01,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:02,024 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:56:02,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:03,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:04,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:05,121 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:05,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:05,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:56:05,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:56:06,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:07,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:08,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:09,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:10,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:11,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:11,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:12,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:13,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:14,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:15,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:16,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:16,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:17,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:18,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:19,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:20,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:20,996 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:56:20,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:56:21,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:22,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:22,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:23,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:24,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:25,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:26,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:27,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:27,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:28,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:29,022 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:56:29,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:30,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:31,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:56:31,072 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:56:31,072 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:56:31,077 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:56:31,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:32,051 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:56:32,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:33,111 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:33,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:34,052 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:56:34,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:35,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:35,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:56:35,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:56:36,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:37,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:38,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:38,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:39,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:40,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:41,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:42,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:43,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:43,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:44,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:45,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:46,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:47,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:48,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:48,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:49,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:50,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:50,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:56:50,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:56:51,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:52,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:53,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:54,082 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:56:54,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:55,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:56,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:57,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:58,884 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:56:59,032 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:56:59,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:00,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:00,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:01,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:02,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:03,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:04,242 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:57:04,243 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:57:04,244 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:57:04,249 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:57:04,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:05,084 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:57:05,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:05,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:05,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:57:05,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:57:06,085 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:57:06,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:07,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:08,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:09,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:10,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:11,128 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:11,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:12,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:13,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:14,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:15,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:17,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:17,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:18,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:19,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:20,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:20,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:57:20,997 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:57:21,057 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:22,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:23,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:23,102 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:24,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:25,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:26,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:27,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:28,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:28,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:29,038 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:57:29,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:30,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:31,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:32,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:33,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:33,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:34,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:34,381 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:57:34,382 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:57:34,382 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:57:34,387 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:57:35,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:35,120 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:57:35,998 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:57:35,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:57:36,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:36,121 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:57:37,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:38,063 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:38,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:39,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:40,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:41,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:42,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:43,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:44,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:44,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:45,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:46,064 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:47,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:48,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:49,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:49,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:50,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:50,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:57:50,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:57:51,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:52,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:53,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:54,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:57:54,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:55,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:56,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:57,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:58,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:57:59,032 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:57:59,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:00,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:00,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:01,175 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:02,178 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:03,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:04,179 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:05,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:05,139 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:58:05,140 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:58:05,140 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:58:05,144 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:58:05,149 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:58:05,545 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:05,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:58:05,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:58:06,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:07,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:08,152 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:58:08,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:09,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:10,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:10,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:11,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:12,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:13,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:14,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:15,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:16,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:16,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:17,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:18,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:19,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:20,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:20,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:58:20,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:58:21,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:22,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:22,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:23,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:24,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:25,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:26,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:27,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:27,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:28,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:29,049 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:58:29,349 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:30,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:31,353 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:32,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:33,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:33,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:34,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:34,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:58:34,402 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:58:34,402 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:58:34,404 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:58:35,179 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:58:35,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:35,997 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:58:35,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:58:36,180 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:58:36,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:37,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:38,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:38,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:39,355 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:40,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:41,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:42,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:43,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:44,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:44,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:45,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:46,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:47,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:48,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:49,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:50,134 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:50,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:50,998 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:58:50,998 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:58:51,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:52,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:53,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:54,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:55,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:56,135 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:58:56,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:57,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:58,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:58:59,041 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:58:59,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:00,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:01,149 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:01,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:02,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:03,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:04,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:05,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:05,682 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:59:05,684 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:59:05,684 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:59:05,687 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:59:06,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:59:06,092 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:59:06,210 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:59:06,246 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:06,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:07,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:08,211 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:59:08,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:09,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:10,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:11,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:12,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:12,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:13,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:14,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:15,361 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:16,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:17,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:18,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:18,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:19,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:20,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:21,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:59:21,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:59:21,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:22,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:23,186 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:23,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:24,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:25,362 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:26,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:27,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:28,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:29,045 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:59:29,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:29,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:30,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:31,363 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:32,372 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:33,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:34,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:34,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:35,376 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:36,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:59:36,098 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:59:36,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:37,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:37,398 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 03:59:37,400 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 03:59:37,401 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 03:59:37,406 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 03:59:38,236 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 03:59:38,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:39,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:39,440 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:40,237 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 03:59:40,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:41,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:42,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:43,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:44,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:45,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:45,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:46,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:47,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:48,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:49,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:50,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:50,511 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:51,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 03:59:51,093 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 03:59:51,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:52,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:53,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:54,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:55,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 03:59:55,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:56,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:57,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:58,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 03:59:59,049 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 03:59:59,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:00,513 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:01,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:01,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:02,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:03,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:04,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:05,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:06,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:00:06,093 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:00:06,172 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:06,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:06,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:00:06,752 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:00:06,753 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:00:06,755 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:00:07,263 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:00:07,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:08,264 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:00:08,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:09,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:10,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:11,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:12,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:12,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:13,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:14,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:15,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:16,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:17,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:17,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:18,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:19,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:20,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:21,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:00:21,094 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:00:21,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:22,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:22,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:23,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:24,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:25,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:26,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:27,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:28,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:28,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:29,053 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:00:29,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:30,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:31,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:32,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:33,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:33,522 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:34,522 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:35,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:36,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:00:36,099 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:00:36,536 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:37,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:38,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:38,359 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:00:38,360 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:00:38,361 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:00:38,363 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:00:38,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:39,291 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:00:39,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:40,291 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:00:40,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:41,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:42,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:43,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:44,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:44,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:45,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:46,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:47,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:48,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:49,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:49,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:50,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:51,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:00:51,097 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:00:51,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:52,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:53,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:54,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:00:54,547 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:55,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:56,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:57,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:58,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:00:59,057 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:00:59,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:00,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:00,548 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:01,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:04,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:04,365 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:01:05,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:10,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:11,580 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:12,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:12,551 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:01:12,552 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:01:13,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:15,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:19,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: keepalive +2025-01-02 04:01:20,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:21,715 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:26,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:29,061 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:01:29,894 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:30,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:31,944 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:36,945 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:37,349 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:01:37,350 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:01:38,350 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:39,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:42,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:46,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:47,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:48,397 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:01:48,398 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:01:49,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:52,519 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:56,822 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:01:57,520 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:01:59,066 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:02:02,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:02,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:02:02,597 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:02:02,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:03,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:04,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:04,636 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:02:05,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:06,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:06,637 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:02:07,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:07,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:08,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:09,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:10,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:11,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:12,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:13,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:13,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:14,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:15,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:16,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:17,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:02:17,594 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:02:17,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:18,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:18,661 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:19,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:20,596 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:21,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:22,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:23,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:24,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:24,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:25,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:26,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:27,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:28,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:29,069 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:02:29,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:30,072 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:30,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:31,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:32,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:02:32,594 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:02:32,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:33,597 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:34,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:35,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:35,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:36,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:37,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:38,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:39,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:40,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:41,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:41,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:42,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:43,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:44,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:45,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:46,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:46,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:47,594 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:02:47,595 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:02:47,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:48,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:49,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:50,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:51,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:51,734 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:52,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:53,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:54,600 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:02:57,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:02:59,072 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:03:03,074 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:08,075 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:13,078 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:16,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:16,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:03:16,095 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:03:17,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:18,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:18,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:19,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:20,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:21,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:22,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:23,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:24,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:24,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:25,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:26,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:27,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:28,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:29,076 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:03:29,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:30,078 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:30,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:31,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:03:31,090 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:03:31,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:32,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:33,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:34,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:35,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:35,244 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:36,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:37,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:38,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:39,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:40,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:41,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:41,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:42,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:43,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:44,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:45,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:46,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:03:46,090 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:03:46,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:46,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:47,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:48,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:49,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:50,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:51,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:52,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:52,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:53,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:54,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:55,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:56,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:57,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:03:57,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:58,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:03:59,080 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:03:59,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:00,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:01,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:04:01,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:04:01,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:02,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:02,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:03,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:04,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:05,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:06,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:07,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:08,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:08,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:09,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:10,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:11,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:12,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:13,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:13,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:14,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:15,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:16,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:04:16,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:04:16,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:17,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:18,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:18,259 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:19,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:20,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:21,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:22,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:23,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:24,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:24,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:25,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:26,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:27,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:28,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:29,084 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:04:29,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:30,086 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:30,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:31,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:04:31,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:04:31,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:32,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:33,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:34,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:35,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:35,255 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:36,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:37,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:38,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:39,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:40,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:41,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:41,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:42,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:43,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:44,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:45,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:46,090 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:04:46,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:04:46,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:46,304 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:47,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:48,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:49,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:50,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:51,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:52,093 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:52,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:53,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:54,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:55,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:56,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:57,100 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:04:58,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:04:59,093 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:05:03,096 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:08,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:13,097 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:18,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:23,098 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:28,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:29,092 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:05:34,095 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:38,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:38,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:05:38,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:05:39,740 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:39,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:40,740 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:42,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:42,864 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:05:43,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:43,865 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:05:44,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:45,066 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:45,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:46,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:46,868 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:05:47,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:48,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:49,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:50,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:50,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:51,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:52,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:53,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:53,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:05:53,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:05:54,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:55,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:55,881 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:05:56,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:57,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:58,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:05:59,096 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:05:59,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:00,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:01,099 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:01,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:02,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:03,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:04,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:05,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:06,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:06,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:07,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:08,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:08,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:06:08,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:06:09,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:10,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:11,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:11,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:12,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:13,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:13,247 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:06:13,249 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:06:13,249 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:06:13,255 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:06:13,893 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:06:14,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:14,894 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:06:15,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:16,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:17,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:17,749 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:18,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:19,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:20,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:21,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:22,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:23,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:23,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:06:23,743 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:06:23,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:24,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:25,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:26,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:27,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:28,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:29,101 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:06:29,103 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:29,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:30,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:31,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:32,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:33,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:34,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:34,749 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:35,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:36,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:37,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:38,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:38,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:06:38,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:06:39,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:39,889 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:40,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:41,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:42,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:43,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:44,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:45,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:45,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:46,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:47,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:47,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:06:47,175 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:06:47,175 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:06:47,178 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:06:47,927 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:06:48,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:48,928 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:06:49,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:50,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:51,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:51,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:52,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:53,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:53,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:06:53,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:06:54,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:55,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:56,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:56,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:06:57,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:58,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:06:59,104 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:06:59,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:00,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:01,143 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:02,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:02,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:03,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:04,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:05,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:06,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:07,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:07,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:08,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:08,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:07:08,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:07:09,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:10,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:11,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:12,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:12,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:13,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:14,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:15,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:16,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:17,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:18,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:18,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:19,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:20,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:20,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:07:20,531 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:07:20,531 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:07:20,534 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:07:20,957 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:07:21,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:22,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:22,959 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:07:23,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:23,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:07:23,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:07:23,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:24,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:25,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:26,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:27,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:28,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:29,109 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:07:29,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:29,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:30,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:31,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:32,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:33,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:34,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:34,758 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:35,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:36,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:37,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:38,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:38,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:07:38,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:07:39,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:39,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:40,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:41,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:42,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:43,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:44,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:45,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:45,762 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:46,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:47,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:48,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:48,568 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:07:48,569 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:07:48,569 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:07:48,570 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:07:48,981 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:07:49,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:50,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:50,959 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:50,983 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:07:51,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:52,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:53,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:53,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:07:53,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:07:54,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:55,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:56,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:56,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:07:57,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:58,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:07:59,112 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:07:59,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:00,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:01,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:02,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:02,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:03,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:04,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:05,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:06,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:07,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:07,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:08,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:08,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:08:08,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:08:09,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:10,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:11,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:12,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:12,972 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:13,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:14,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:15,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:16,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:08:16,025 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:08:16,026 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:08:16,028 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:08:16,473 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:17,018 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:08:17,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:18,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:18,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:19,019 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:08:19,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:20,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:21,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:22,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:23,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:23,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:08:23,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:08:23,891 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:24,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:25,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:26,169 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:27,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:28,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:28,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:29,116 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:08:29,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:30,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:31,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:32,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:33,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:34,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:34,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:35,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:36,170 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:37,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:38,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:38,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:08:38,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:08:39,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:40,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:40,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:41,176 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:42,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:43,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:44,174 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:45,173 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:45,966 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:46,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:08:46,031 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:08:46,031 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:08:46,033 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:08:46,060 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:08:46,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:47,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:48,427 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:49,062 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:08:49,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:50,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:51,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:51,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:52,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:53,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:53,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:08:53,744 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:08:54,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:55,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:56,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:57,428 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:57,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:08:58,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:08:59,120 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:08:59,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:00,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:01,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:02,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:03,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:03,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:04,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:05,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:06,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:07,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:08,429 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:08,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:09:08,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:09:08,912 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:09,430 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:10,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:11,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:12,433 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:13,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:13,966 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:14,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:15,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:15,965 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:09:15,967 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:09:15,967 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:09:15,969 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:09:16,095 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:09:16,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:17,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:18,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:18,968 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:19,097 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:09:19,437 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:20,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:21,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:22,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:23,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:23,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:09:23,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:09:24,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:24,966 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:25,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:26,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:27,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:28,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:29,124 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:09:29,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:30,126 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:30,438 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:31,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:32,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:33,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:34,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:35,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:35,967 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:36,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:37,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:38,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:38,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:09:38,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:09:39,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:40,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:41,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:41,972 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:42,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:43,488 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:44,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:45,487 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:46,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:46,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:47,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:09:47,481 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:09:47,482 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:09:47,484 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:09:47,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:48,124 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:09:48,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:49,125 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:09:49,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:50,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:51,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:52,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:52,971 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:53,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:53,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:09:53,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:09:54,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:55,624 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:56,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:57,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:58,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:09:58,971 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:09:59,126 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:09:59,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:00,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:01,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:02,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:03,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:04,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:04,971 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:05,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:06,625 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:07,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:08,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:08,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:10:08,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:10:09,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:09,971 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:10,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:11,626 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:12,632 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:13,629 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:14,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:14,978 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:15,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:16,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:16,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:10:16,665 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:10:16,666 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:10:16,668 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:10:17,150 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:10:17,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:18,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:19,151 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:10:19,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:20,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:20,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:21,633 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:22,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:23,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:23,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:10:23,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:10:24,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:25,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:26,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:26,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:27,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:28,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:29,130 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:10:29,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:30,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:31,634 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:32,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:32,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:33,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:34,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:35,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:36,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:37,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:38,018 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:38,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:38,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:10:38,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:10:39,635 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:40,636 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:41,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:42,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:43,024 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:43,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:44,646 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:45,639 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:46,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:46,833 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:10:46,835 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:10:46,835 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:10:46,837 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:10:47,178 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:10:47,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:48,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:48,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:49,180 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:10:49,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:50,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:51,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:52,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:53,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:53,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:10:53,745 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:10:54,018 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:54,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:55,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:56,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:57,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:58,640 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:10:59,134 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:10:59,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:10:59,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:00,783 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:01,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:02,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:03,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:04,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:05,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:05,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:06,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:07,641 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:08,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:08,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:11:08,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:11:09,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:10,642 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:11,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:11,645 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:12,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:13,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:14,649 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:15,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:16,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:16,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:17,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:11:17,358 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:11:17,359 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:11:17,360 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:11:17,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:18,206 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:11:18,771 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:19,208 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:11:19,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:20,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:21,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:22,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:22,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:23,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:11:23,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:11:23,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:24,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:25,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:26,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:27,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:28,019 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:28,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:29,138 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:11:29,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:30,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:31,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:32,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:33,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:33,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:34,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:35,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:36,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:37,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:38,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:11:38,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:11:38,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:38,913 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:39,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:40,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:41,842 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:42,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:43,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:44,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:44,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:45,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:46,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:47,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:11:47,054 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:11:47,055 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:11:47,057 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:11:47,241 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:11:47,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:48,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:49,242 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:11:49,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:50,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:50,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:51,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:52,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:53,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:11:53,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:11:53,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:54,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:55,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:56,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:11:56,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:57,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:58,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:11:59,142 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:11:59,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:00,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:01,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:01,849 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:02,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:03,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:04,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:05,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:06,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:07,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:07,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:08,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:12:08,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:12:08,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:09,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:10,850 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:11,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:12,030 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:12,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:13,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:14,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:15,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:16,861 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:17,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:17,858 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:17,947 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:12:17,948 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:12:17,948 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:12:17,949 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:12:18,271 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:12:18,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:19,272 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:12:19,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:20,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:21,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:22,031 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:22,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:23,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:12:23,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:12:23,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:24,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:25,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:26,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:27,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:28,031 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:28,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:29,146 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:12:29,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:30,860 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:32,094 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:33,091 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:33,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:34,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:35,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:36,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:37,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:38,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:38,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:12:38,746 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:12:38,815 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:39,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:40,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:41,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:42,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:43,092 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:44,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:44,101 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:45,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:46,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:47,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:48,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:49,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:49,727 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:12:49,728 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:12:49,728 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:12:49,728 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:49,729 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:12:50,105 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:50,299 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:12:51,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:51,300 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:12:52,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:53,106 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:53,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:12:53,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:12:54,106 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:55,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:12:55,106 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:56,106 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:57,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:58,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:59,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:12:59,149 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:13:00,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:00,151 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:01,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:02,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:03,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:04,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:05,107 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:06,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:06,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:07,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:08,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:08,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:13:08,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:13:09,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:10,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:11,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:12,036 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:12,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:13,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:14,108 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:15,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:16,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:17,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:17,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:18,112 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:19,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:20,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:21,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:21,159 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:13:21,160 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:13:21,161 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:13:21,162 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:13:21,329 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:13:22,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:22,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:23,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:23,331 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:13:23,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:13:23,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:13:24,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:25,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:26,113 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:27,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:28,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:28,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:29,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:29,152 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:13:30,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:31,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:32,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:33,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:33,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:34,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:35,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:36,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:37,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:38,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:38,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:13:38,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:13:38,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:39,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:40,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:41,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:42,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:43,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:44,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:44,115 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:45,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:46,121 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:47,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:48,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:49,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:49,124 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:50,122 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:51,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:51,951 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:13:51,952 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:13:51,953 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:13:51,955 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:13:52,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:52,361 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:13:53,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:53,362 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:13:53,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:13:53,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:13:54,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:54,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:55,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:56,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:57,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:58,357 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:13:59,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:13:59,156 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:13:59,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:00,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:01,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:02,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:03,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:04,358 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:05,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:05,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:06,390 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:07,390 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:08,390 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:08,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:14:08,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:14:09,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:10,390 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:11,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:11,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:12,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:13,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:14,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:15,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:16,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:16,391 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:17,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:18,395 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:19,395 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:20,395 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:21,055 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:21,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:22,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:22,788 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:14:22,790 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:14:22,791 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:14:22,792 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:14:23,390 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:14:23,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:23,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:14:23,747 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:14:24,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:25,392 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:14:25,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:26,399 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:27,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:27,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:28,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:29,160 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:14:29,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:30,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:31,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:32,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:32,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:33,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:34,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:35,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:36,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:37,400 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:38,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:38,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:38,748 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:14:38,748 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:14:39,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:40,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:41,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:42,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:43,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:44,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:44,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:45,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:46,401 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:47,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:48,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:49,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:49,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:50,402 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:51,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:52,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:53,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:53,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:14:53,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:14:54,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:54,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:55,417 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:56,410 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:56,804 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:14:56,805 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:14:56,805 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:14:56,807 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:14:57,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:57,423 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:14:58,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:59,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:14:59,164 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:14:59,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:14:59,425 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:15:00,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:01,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:02,412 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:03,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:04,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:04,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:05,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:06,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:07,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:08,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:08,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:15:08,752 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:15:09,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:10,052 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:10,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:11,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:12,413 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:13,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:14,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:15,053 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:15,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:16,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:17,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:18,414 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:19,416 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:20,069 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:20,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:21,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:22,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:23,426 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:23,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:15:23,752 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:15:24,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:24,489 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:15:24,490 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:15:24,490 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:15:24,492 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:15:25,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:25,459 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:15:25,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:26,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:27,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:27,461 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:15:28,423 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:29,168 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:15:29,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:30,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:31,171 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:31,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:32,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:33,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:34,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:35,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:36,424 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:37,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:37,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:38,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:38,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:15:38,752 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:15:39,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:40,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:41,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:42,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:43,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:43,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:44,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:45,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:46,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:47,463 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:48,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:48,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:49,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:50,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:51,465 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:52,465 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:53,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:15:53,155 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:15:53,155 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:15:53,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:53,157 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:15:53,498 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:15:53,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:53,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:15:53,752 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:15:54,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:55,500 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:15:55,556 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:56,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:57,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:58,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:15:59,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:15:59,171 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:15:59,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:00,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:01,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:02,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:03,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:04,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:05,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:05,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:06,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:07,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:08,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:08,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:16:08,753 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:16:09,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:10,561 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:11,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:11,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:12,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:13,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:14,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:15,558 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:16,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:17,056 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:17,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:18,559 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:19,562 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:20,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:21,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:22,062 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:22,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:23,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:23,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:16:23,753 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:16:24,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:16:24,039 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:16:24,040 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:16:24,041 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:16:24,537 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:16:24,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:25,538 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:16:25,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:26,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:27,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:27,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:28,566 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:29,176 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:16:29,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:30,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:31,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:32,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:33,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:33,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:34,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:35,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:36,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:37,567 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:38,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:38,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:16:38,754 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:16:38,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:39,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:40,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:41,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:42,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:43,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:44,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:44,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:45,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:46,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:47,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:48,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:49,187 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:49,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:50,667 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:51,679 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:52,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:53,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:53,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:16:53,762 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:16:54,197 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:54,673 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:55,678 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:56,674 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:57,408 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:16:57,410 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:16:57,410 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:16:57,412 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:16:57,570 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:16:57,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:58,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:16:59,180 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:16:59,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:16:59,571 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:16:59,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:00,816 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:01,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:02,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:03,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:04,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:04,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:05,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:06,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:07,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:08,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:17:08,759 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:17:08,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:09,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:10,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:10,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:11,817 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:12,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:13,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:14,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:15,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:15,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:16,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:17,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:18,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:19,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:20,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:21,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:21,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:22,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:23,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:17:23,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:17:23,819 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:24,825 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:25,827 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:26,212 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:26,833 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:27,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:28,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:29,182 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:17:29,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:29,981 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:17:29,982 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:17:29,983 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:17:29,985 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:17:30,612 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:17:30,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:31,613 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:17:31,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:32,023 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:32,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:33,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:34,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:35,830 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:36,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:37,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:37,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:38,759 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:17:38,760 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:17:38,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:39,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:40,831 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:41,903 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:42,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:43,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:43,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:44,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:45,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:46,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:47,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:48,199 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:48,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:49,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:50,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:51,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:52,902 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:53,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:53,768 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:17:53,777 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:17:53,912 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:54,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:55,913 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:56,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:57,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:58,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:17:58,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:17:59,184 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:17:59,436 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:17:59,438 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:17:59,439 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:17:59,440 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:17:59,642 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:17:59,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:00,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:01,644 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:18:01,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:02,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:03,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:03,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:04,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:05,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:06,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:07,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:08,762 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:18:08,762 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:18:08,892 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:08,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:09,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:10,911 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:12,072 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:13,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:14,059 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:14,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:15,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:16,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:17,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:18,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:19,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:20,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:20,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:21,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:22,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:23,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:23,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:18:23,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:18:24,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:25,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:25,211 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:26,068 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:27,067 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:28,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:29,065 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:29,192 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:18:29,725 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:18:29,726 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:18:29,727 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:18:29,728 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:18:30,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:30,681 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:18:30,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:31,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:31,682 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:18:32,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:33,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:34,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:35,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:36,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:36,203 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:37,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:38,144 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:38,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:18:38,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:18:39,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:40,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:41,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:42,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:42,203 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:43,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:44,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:45,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:46,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:47,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:47,203 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:48,145 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:49,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:50,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:51,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:52,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:53,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:53,203 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:53,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:18:53,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:18:54,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:55,156 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:56,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:57,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:58,155 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:58,212 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:18:59,152 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:18:59,196 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:19:00,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:00,435 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:19:00,437 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:19:00,438 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:19:00,439 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:19:00,711 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:19:01,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:01,712 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:19:02,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:03,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:03,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:04,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:05,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:06,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:07,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:08,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:08,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:19:08,767 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:19:08,870 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:09,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:10,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:11,153 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:12,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:13,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:14,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:14,204 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:15,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:16,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:17,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:18,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:19,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:20,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:20,204 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:21,154 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:22,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:23,158 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:23,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:19:23,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:19:24,165 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:25,162 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:25,206 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:26,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:27,163 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:27,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:19:27,922 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:19:27,922 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:19:27,923 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:19:28,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:28,738 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:19:29,200 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:19:29,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:29,739 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:19:30,208 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:30,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:31,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:32,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:33,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:34,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:35,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:36,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:36,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:37,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:38,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:38,768 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:19:38,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:19:39,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:40,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:41,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:42,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:42,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:43,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:44,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:45,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:46,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:47,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:47,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:48,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:49,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:50,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:51,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:52,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:52,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:53,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:53,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:19:53,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:19:54,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:55,344 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:56,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:57,229 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:19:57,345 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:58,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:59,204 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:19:59,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:19:59,897 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:19:59,899 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:19:59,899 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:19:59,901 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:20:00,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:00,775 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:20:01,350 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:01,776 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:20:02,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:02,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:03,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:04,346 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:05,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:06,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:07,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:08,213 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:08,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:08,769 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:20:08,769 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:20:09,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:10,347 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:11,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:12,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:13,213 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:13,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:14,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:15,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:16,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:17,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:18,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:19,213 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:19,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:20,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:21,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:22,648 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:23,656 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:23,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:20:23,776 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:20:24,217 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:24,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:25,657 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:26,654 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:27,654 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:28,654 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:29,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:20:29,051 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:20:29,051 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:20:29,053 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:20:29,208 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:20:29,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:29,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:29,807 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:20:29,808 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:20:30,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:31,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:32,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:33,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:34,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:34,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:35,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:36,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:37,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:38,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:38,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:20:38,772 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:20:39,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:40,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:40,659 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:41,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:42,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:43,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:44,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:45,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:45,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:46,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:47,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:48,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:49,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:50,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:51,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:51,660 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:52,661 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:53,661 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:53,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:20:53,772 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:20:54,661 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:55,668 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:56,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:20:56,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:57,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:58,664 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:20:59,213 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:20:59,665 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:00,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:01,220 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:21:01,222 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:21:01,222 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:21:01,225 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:21:01,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:01,844 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:21:01,844 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:21:02,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:02,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:03,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:04,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:05,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:06,669 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:07,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:07,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:08,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:08,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:21:08,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:21:09,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:10,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:11,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:12,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:13,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:13,670 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:14,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:15,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:16,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:17,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:18,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:18,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:19,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:20,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:21,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:22,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:23,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:21:23,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:21:23,813 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:23,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:24,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:25,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:26,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:27,779 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:28,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:29,219 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:21:29,222 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:29,779 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:30,784 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:31,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:31,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:21:31,909 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:21:31,909 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:21:31,911 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:21:32,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:32,880 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:21:33,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:33,881 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:21:34,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:34,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:35,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:36,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:37,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:38,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:21:38,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:21:38,812 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:39,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:40,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:40,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:41,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:42,780 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:43,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:44,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:45,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:46,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:46,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:47,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:48,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:49,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:50,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:51,466 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:51,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:52,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:53,772 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:21:53,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:21:53,813 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:54,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:55,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:56,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:57,469 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:21:57,782 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:58,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:21:59,219 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:21:59,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:00,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:01,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:02,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:02,871 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:22:02,872 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:22:02,872 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:22:02,873 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:02,874 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:22:02,909 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:22:03,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:03,910 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:22:04,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:05,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:06,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:07,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:08,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:08,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:22:08,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:22:08,813 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:09,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:10,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:11,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:12,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:13,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:14,467 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:14,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:15,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:16,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:17,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:18,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:19,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:19,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:20,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:21,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:22,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:23,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:22:23,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:22:23,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:24,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:25,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:25,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:26,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:27,859 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:28,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:29,225 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:22:29,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:30,473 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:30,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:31,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:32,853 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:33,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:22:33,350 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:22:33,351 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:22:33,352 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:22:33,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:33,939 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:22:33,940 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:22:34,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:35,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:36,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:36,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:37,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:38,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:22:38,773 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:22:38,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:39,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:40,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:41,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:42,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:42,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:43,854 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:44,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:45,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:46,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:47,470 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:47,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:48,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:49,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:50,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:51,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:52,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:53,471 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:53,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:22:53,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:22:53,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:54,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:55,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:56,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:57,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:58,480 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:22:58,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:22:59,229 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:22:59,864 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:00,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:01,379 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:23:01,380 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:23:01,380 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:23:01,382 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:23:01,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:01,972 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:23:01,973 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:23:02,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:03,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:04,420 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:04,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:05,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:06,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:07,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:08,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:23:08,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:23:08,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:09,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:09,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:10,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:11,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:12,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:13,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:14,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:14,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:15,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:16,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:17,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:18,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:19,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:19,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:20,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:21,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:22,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:23,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:23:23,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:23:23,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:24,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:25,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:25,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:26,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:27,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:28,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:29,232 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:23:29,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:30,486 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:30,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:31,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:32,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:33,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:34,089 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:23:34,091 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:23:34,091 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:23:34,093 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:23:34,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:35,005 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:23:35,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:35,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:36,006 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:23:36,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:37,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:38,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:23:38,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:23:38,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:39,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:40,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:41,501 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:41,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:42,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:43,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:44,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:45,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:46,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:47,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:47,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:48,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:49,875 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:50,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:51,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:52,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:52,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:53,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:23:53,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:23:53,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:54,890 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:55,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:56,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:57,508 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:23:57,884 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:58,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:23:59,235 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:23:59,880 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:00,555 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:24:00,556 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:24:00,557 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:24:00,560 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:24:00,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:01,036 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:24:01,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:02,037 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:24:02,598 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:02,962 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:03,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:04,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:05,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:06,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:07,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:08,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:08,773 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:24:08,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:24:08,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:09,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:10,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:11,963 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:12,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:13,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:13,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:14,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:15,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:16,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:17,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:18,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:18,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:19,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:20,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:21,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:22,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:23,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:23,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:24:23,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:24:23,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:24,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:25,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:26,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:27,975 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:28,510 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:28,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:29,241 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:24:29,970 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:30,909 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:24:30,911 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:24:30,911 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:24:30,916 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:24:31,072 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:24:31,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:32,073 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:24:32,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:33,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:33,950 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:34,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:35,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:36,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:37,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:38,319 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:38,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:24:38,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:24:39,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:39,507 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:40,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:41,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:42,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:43,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:44,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:44,507 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:45,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:46,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:47,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:48,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:49,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:49,507 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:50,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:51,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:52,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:53,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:53,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:24:53,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:24:54,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:54,507 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:24:55,321 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:56,325 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:57,325 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:58,330 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:59,245 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:24:59,334 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:24:59,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:00,325 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:00,876 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:25:00,877 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:25:00,878 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:25:00,879 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:25:01,104 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:25:01,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:02,105 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:25:02,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:03,325 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:04,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:05,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:05,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:06,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:07,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:08,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:08,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:25:08,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:25:09,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:10,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:11,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:11,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:12,326 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:13,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:14,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:15,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:16,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:16,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:17,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:18,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:19,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:20,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:21,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:22,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:22,512 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:23,327 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:23,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:25:23,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:25:24,328 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:25,330 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:26,330 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:27,335 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:27,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:28,335 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:29,248 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:25:29,343 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:30,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:30,514 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:25:30,516 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:25:30,516 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:25:30,517 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:25:31,134 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:25:31,338 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:32,135 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:25:32,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:32,557 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:33,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:34,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:35,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:36,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:37,339 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:38,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:38,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:38,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:25:38,774 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:25:39,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:40,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:41,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:42,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:43,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:43,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:44,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:45,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:46,340 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:47,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:48,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:49,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:49,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:50,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:51,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:52,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:53,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:53,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:25:53,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:25:54,341 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:55,356 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:55,526 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:25:56,351 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:57,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:58,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:25:59,249 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:25:59,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:00,348 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:00,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:01,070 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:26:01,071 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:26:01,072 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:26:01,073 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:26:01,164 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:26:01,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:02,165 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:26:02,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:03,502 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:04,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:05,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:05,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:06,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:07,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:08,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:08,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:26:08,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:26:09,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:10,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:11,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:11,524 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:12,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:13,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:14,503 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:15,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:16,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:16,524 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:17,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:18,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:19,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:20,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:21,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:22,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:22,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:23,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:23,774 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:26:23,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:26:24,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:25,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:26,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:27,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:28,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:28,525 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:29,255 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:26:29,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:30,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:31,510 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:32,511 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:33,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:33,528 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:34,518 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:35,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:36,521 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:37,315 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:26:37,316 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:26:37,317 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:26:37,320 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:26:37,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:38,199 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:26:38,200 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:26:38,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:38,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:26:38,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:26:38,931 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:39,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:40,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:41,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:42,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:43,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:44,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:44,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:45,737 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:46,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:47,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:48,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:49,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:49,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:50,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:51,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:52,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:53,738 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:53,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:26:53,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:26:54,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:26:54,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:55,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:56,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:57,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:58,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:26:59,258 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:26:59,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:00,529 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:00,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:01,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:02,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:03,739 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:04,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:05,535 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:05,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:06,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:07,743 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:08,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:08,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:27:08,775 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:27:09,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:27:09,211 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:27:09,212 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:27:09,218 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:27:09,238 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:27:09,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:10,239 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:27:10,650 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:10,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:11,752 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:12,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:13,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:14,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:15,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:16,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:16,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:17,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:18,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:19,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:20,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:21,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:22,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:22,753 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:23,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:23,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:27:23,776 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:27:24,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:25,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:26,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:27,647 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:27,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:28,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:29,262 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:27:29,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:30,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:31,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:32,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:33,652 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:33,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:34,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:35,761 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:36,763 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:37,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:38,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:38,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:27:38,782 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:27:38,863 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:39,418 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:27:39,420 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:27:39,421 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:27:39,426 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:27:39,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:40,273 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:27:40,273 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:27:40,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:41,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:42,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:43,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:44,836 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:44,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:45,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:46,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:47,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:48,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:49,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:50,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:50,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:51,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:52,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:53,778 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:27:53,779 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:27:53,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:54,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:55,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:27:55,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:56,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:57,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:58,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:27:59,266 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:27:59,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:00,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:01,837 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:01,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:02,847 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:03,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:04,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:05,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:06,845 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:06,858 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:07,852 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:08,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:28:08,781 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:28:08,858 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:09,862 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:10,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:11,872 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:12,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:28:12,755 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:28:12,755 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:28:12,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:12,761 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:28:13,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:13,315 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:28:13,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:14,316 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:28:14,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:15,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:16,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:17,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:17,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:18,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:19,899 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:20,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:21,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:22,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:23,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:28:23,781 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:28:23,855 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:23,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:24,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:25,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:26,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:27,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:28,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:29,270 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:28:29,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:29,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:30,900 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:31,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:32,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:33,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:34,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:34,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:35,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:36,901 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:37,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:38,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:28:38,781 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:28:38,910 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:39,840 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:39,904 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:40,906 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:41,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:42,905 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:43,792 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:28:43,794 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:28:43,794 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:28:43,797 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:28:44,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:44,356 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:28:44,357 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:28:44,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:45,232 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:46,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:47,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:48,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:49,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:50,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:50,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:51,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:52,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:53,234 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:53,781 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:28:53,782 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:28:54,233 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:55,461 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:55,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:28:56,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:57,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:58,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:28:59,275 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:28:59,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:00,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:01,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:01,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:02,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:03,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:04,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:05,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:06,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:06,841 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:07,368 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:08,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:08,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:29:08,786 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:29:09,375 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:10,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:11,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:11,843 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:12,374 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:13,042 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:29:13,044 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:29:13,045 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:29:13,047 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:29:13,393 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:29:13,450 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:14,395 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:29:14,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:15,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:16,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:16,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:17,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:18,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:19,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:20,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:21,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:22,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:22,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:23,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:23,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:29:23,786 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:29:24,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:25,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:26,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:27,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:27,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:28,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:29,279 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:29:29,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:30,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:31,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:32,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:32,844 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:33,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:34,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:35,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:36,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:37,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:37,848 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:38,462 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:38,793 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:29:38,797 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:29:39,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:40,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:41,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:42,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:42,766 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:29:42,768 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:29:42,768 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:29:42,770 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:29:43,428 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:29:43,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:43,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:44,429 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:29:44,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:45,459 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:46,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:47,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:48,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:48,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:49,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:50,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:51,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:52,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:53,460 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:53,791 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:29:53,791 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:29:53,912 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:54,588 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:55,498 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:56,498 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:57,498 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:58,498 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:29:59,283 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:29:59,285 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:29:59,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:00,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:01,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:02,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:03,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:04,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:04,846 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:05,499 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:06,504 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:07,505 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:08,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:08,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:30:08,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:30:09,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:09,858 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:10,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:11,509 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:12,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:30:12,193 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:30:12,193 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:30:12,194 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:30:12,460 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:30:12,604 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:13,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:14,461 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:30:14,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:15,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:15,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:16,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:17,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:18,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:19,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:20,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:21,601 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:21,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:22,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:23,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:23,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:30:23,794 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:30:24,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:25,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:26,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:27,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:27,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:28,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:29,287 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:30:29,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:30,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:31,602 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:32,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:32,851 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:33,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:34,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:35,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:36,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:37,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:38,603 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:38,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:30:38,794 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:30:38,961 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:39,604 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:40,613 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:41,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:42,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:43,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:44,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:44,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:45,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:46,121 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:30:46,122 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:30:46,123 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:30:46,125 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:30:46,496 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:30:46,497 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:30:46,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:47,607 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:48,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:49,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:49,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:50,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:51,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:52,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:53,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:53,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:30:53,795 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:30:54,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:55,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:55,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:30:56,608 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:57,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:58,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:30:59,291 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:30:59,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:00,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:00,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:01,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:02,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:03,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:04,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:05,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:06,818 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:06,857 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:07,819 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:08,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:31:08,795 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:31:08,834 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:09,828 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:10,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:11,824 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:11,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:12,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:13,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:14,823 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:15,595 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:31:15,596 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:31:15,597 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:31:15,598 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:31:16,008 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:16,534 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:31:16,534 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:31:17,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:17,638 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:18,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:19,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:20,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:21,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:22,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:22,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:23,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:23,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:31:23,795 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:31:24,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:25,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:26,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:27,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:28,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:28,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:29,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:29,295 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:31:30,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:31,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:32,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:33,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:34,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:34,865 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:35,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:36,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:37,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:38,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:38,794 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:31:38,795 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:31:39,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:40,007 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:40,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:41,012 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:42,012 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:43,012 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:44,015 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:45,013 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:46,013 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:46,618 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:31:46,620 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:31:46,621 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:31:46,621 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:46,626 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:31:47,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:47,563 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:31:48,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:48,564 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:31:49,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:50,026 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:51,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:51,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:52,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:53,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:53,795 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:31:53,795 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:31:54,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:55,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:56,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:57,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:57,866 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:31:58,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:59,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:31:59,299 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:32:00,027 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:01,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:02,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:02,867 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:03,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:04,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:05,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:06,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:07,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:08,028 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:08,803 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:32:08,810 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:32:08,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:09,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:10,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:11,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:12,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:13,029 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:14,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:14,775 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:32:14,776 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:32:14,776 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:32:14,776 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:14,777 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:32:15,185 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:15,593 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:32:16,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:17,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:17,595 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:32:18,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:19,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:19,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:20,181 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:21,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:22,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:23,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:23,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:32:23,796 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:32:24,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:25,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:25,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:26,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:27,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:28,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:29,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:29,302 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:32:30,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:30,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:31,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:32,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:33,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:34,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:35,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:36,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:36,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:37,183 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:38,190 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:38,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:32:38,805 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:32:39,195 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:40,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:41,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:41,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:42,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:43,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:44,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:44,245 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:32:44,246 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:32:44,247 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:32:44,249 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:32:44,621 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:32:45,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:45,622 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:32:46,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:47,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:47,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:48,191 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:49,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:50,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:51,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:52,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:53,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:53,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:32:53,798 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:32:53,946 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:32:54,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:55,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:56,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:57,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:58,192 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:59,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:32:59,306 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:32:59,313 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:00,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:01,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:02,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:03,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:04,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:04,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:05,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:06,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:07,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:08,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:08,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:33:08,798 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:33:09,193 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:10,194 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:10,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:11,204 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:12,200 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:13,204 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:14,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:15,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:16,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:16,936 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:17,207 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:17,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:33:17,266 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:33:17,267 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:33:17,268 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:33:17,657 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:33:18,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:19,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:19,658 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:33:20,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:21,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:21,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:22,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:23,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:23,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:33:23,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:33:24,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:25,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:26,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:27,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:27,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:28,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:29,209 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:29,311 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:33:30,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:31,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:32,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:33,072 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:33,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:34,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:35,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:36,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:37,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:38,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:38,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:33:38,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:33:38,920 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:39,210 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:40,211 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:41,211 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:42,217 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:43,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:43,980 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:44,214 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:45,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:46,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:47,218 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:47,964 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:33:47,965 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:33:47,965 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:33:47,967 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:33:48,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:48,688 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:33:49,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:49,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:49,688 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:33:50,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:51,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:52,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:53,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:53,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:33:53,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:33:54,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:54,908 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:33:55,366 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:56,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:57,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:58,367 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:59,315 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:33:59,449 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:33:59,979 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:00,446 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:01,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:02,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:03,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:04,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:04,979 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:05,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:06,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:07,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:08,447 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:08,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:34:08,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:34:09,448 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:09,994 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:10,448 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:11,451 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:12,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:13,453 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:14,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:15,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:15,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:34:15,766 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:34:15,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:34:15,767 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:15,768 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:34:16,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:16,716 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:34:17,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:17,717 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:34:18,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:19,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:20,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:20,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:21,454 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:22,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:23,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:23,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:34:23,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:34:24,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:25,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:26,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:26,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:27,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:28,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:29,319 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:34:29,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:30,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:31,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:31,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:32,455 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:33,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:34,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:35,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:36,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:37,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:37,983 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:38,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:38,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:34:38,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:34:39,456 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:40,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:41,463 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:42,464 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:42,987 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:43,463 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:44,463 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:45,474 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:46,468 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:46,720 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:34:46,721 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:34:46,721 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:34:46,723 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:34:46,750 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:34:47,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:48,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:48,762 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:49,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:49,752 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:34:50,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:51,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:52,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:53,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:53,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:34:53,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:34:53,874 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:54,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:55,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:56,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:57,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:58,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:34:58,984 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:34:59,321 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:34:59,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:00,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:01,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:02,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:03,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:03,984 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:04,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:05,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:06,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:07,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:08,478 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:08,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:35:08,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:35:09,477 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:09,958 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:10,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:11,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:12,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:13,492 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:14,497 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:14,990 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:15,496 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:16,377 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:35:16,378 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:35:16,378 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:35:16,379 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:35:16,782 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:35:16,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:17,783 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:35:17,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:18,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:19,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:20,785 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:20,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:21,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:22,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:23,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:23,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:35:23,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:35:24,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:25,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:26,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:26,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:27,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:28,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:29,325 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:35:29,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:30,786 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:31,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:31,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:32,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:33,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:34,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:35,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:36,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:36,988 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:37,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:38,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:38,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:35:38,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:35:39,787 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:40,790 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:41,795 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:41,994 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:42,795 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:43,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:44,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:45,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:46,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:46,815 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:35:46,816 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:35:46,816 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:35:46,817 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:35:47,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:47,812 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:35:47,856 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:48,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:49,796 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:49,814 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:35:50,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:51,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:52,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:52,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:53,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:53,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:35:53,799 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:35:54,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:55,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:56,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:57,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:57,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:35:58,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:35:59,329 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:35:59,797 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:00,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:01,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:02,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:02,992 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:03,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:04,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:05,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:06,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:07,798 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:08,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:08,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:36:08,800 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:36:08,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:09,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:10,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:11,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:12,805 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:13,808 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:14,003 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:14,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:15,182 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:36:15,184 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:36:15,184 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:36:15,186 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:36:15,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:15,844 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:36:16,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:17,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:17,846 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:36:18,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:19,225 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:19,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:20,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:21,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:22,806 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:23,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:36:23,800 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:36:23,839 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:24,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:24,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:25,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:26,807 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:27,923 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:28,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:29,333 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:36:29,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:29,993 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:30,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:31,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:32,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:33,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:34,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:35,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:35,993 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:36,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:37,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:38,799 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:36:38,800 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:36:38,922 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:39,930 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:40,925 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:41,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:41,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:42,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:43,925 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:44,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:45,415 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:36:45,417 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:36:45,417 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:36:45,419 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:36:45,878 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:36:45,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:46,457 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:46,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:47,880 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:36:47,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:48,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:49,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:50,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:51,927 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:51,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:52,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:53,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:36:53,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:36:53,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:54,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:55,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:56,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:57,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:57,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:36:58,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:36:59,337 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:36:59,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:00,928 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:01,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:02,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:02,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:03,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:04,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:05,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:06,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:07,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:07,999 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:08,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:37:08,800 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:37:08,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:09,929 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:10,942 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:11,934 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:12,937 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:13,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:13,935 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:14,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:15,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:16,835 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:37:16,836 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:37:16,836 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:37:16,838 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:37:16,912 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:37:17,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:18,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:18,877 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:19,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:19,914 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:37:20,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:21,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:22,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:23,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:23,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:37:23,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:37:23,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:24,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:25,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:26,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:27,248 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:28,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:29,003 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:29,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:29,342 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:37:30,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:31,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:32,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:33,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:34,003 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:34,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:35,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:36,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:37,249 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:38,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:38,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:37:38,800 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:37:39,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:39,336 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:40,250 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:41,252 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:42,262 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:43,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:44,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:45,006 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:45,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:46,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:46,764 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:37:46,765 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:37:46,766 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:37:46,767 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:37:46,943 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:37:47,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:48,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:49,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:49,946 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:37:50,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:51,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:51,263 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:52,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:53,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:53,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:37:53,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:37:54,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:55,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:56,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:57,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:37:57,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:58,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:59,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:37:59,346 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:38:00,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:01,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:02,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:02,264 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:03,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:04,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:05,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:06,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:07,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:08,005 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:08,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:08,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:38:08,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:38:09,265 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:10,273 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:11,272 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:12,275 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:13,015 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:13,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:14,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:15,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:15,610 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:38:15,611 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:38:15,612 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:38:15,613 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:38:15,977 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:38:16,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:17,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:17,979 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:38:18,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:18,652 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:19,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:20,282 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:21,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:22,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:23,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:23,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:38:23,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:38:23,878 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:24,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:25,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:26,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:27,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:28,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:29,009 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:29,283 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:29,350 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:38:30,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:31,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:32,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:33,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:34,204 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:34,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:35,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:36,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:37,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:38,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:38,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:38:38,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:38:39,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:39,895 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:40,284 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:41,285 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:42,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:43,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:44,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:45,117 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:45,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:46,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:47,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:47,686 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:38:47,688 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:38:47,688 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:38:47,690 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:38:48,008 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:38:48,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:49,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:50,010 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:38:50,288 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:51,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:51,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:52,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:53,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:53,800 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:38:53,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:38:54,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:55,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:56,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:57,109 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:38:57,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:58,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:59,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:38:59,354 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:39:00,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:01,289 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:02,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:02,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:03,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:04,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:05,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:06,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:07,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:08,110 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:08,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:08,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:39:08,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:39:09,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:10,290 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:11,291 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:12,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:13,114 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:13,295 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:14,298 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:15,305 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:16,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:17,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:17,452 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:39:17,454 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:39:17,454 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:39:17,455 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:39:18,038 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:39:18,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:18,493 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:19,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:20,039 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:39:20,299 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:21,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:22,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:23,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:23,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:39:23,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:39:23,884 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:24,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:25,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:26,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:27,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:28,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:29,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:29,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:29,357 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:39:30,300 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:31,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:32,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:33,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:34,116 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:34,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:35,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:36,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:37,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:38,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:38,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:39:38,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:39:39,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:39,939 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:40,301 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:41,306 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:42,306 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:43,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:44,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:45,118 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:45,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:46,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:47,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:47,530 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:39:47,532 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:39:47,532 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:39:47,534 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:39:48,098 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:39:48,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:49,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:50,100 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:39:50,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:50,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:51,307 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:52,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:53,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:53,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:39:53,801 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:39:54,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:55,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:56,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:39:56,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:57,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:58,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:59,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:39:59,362 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:40:00,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:01,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:01,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:02,308 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:03,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:04,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:05,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:06,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:06,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:07,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:08,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:08,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:40:08,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:40:09,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:10,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:11,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:12,119 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:12,309 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:13,310 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:14,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:15,320 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:16,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:17,131 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:17,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:18,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:19,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:19,517 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:40:19,518 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:40:19,518 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:40:19,519 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:40:20,144 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:40:20,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:21,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:22,146 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:40:22,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:23,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:23,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:23,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:40:23,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:40:24,317 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:25,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:26,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:27,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:28,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:29,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:29,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:29,365 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:40:30,318 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:31,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:32,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:33,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:34,475 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:35,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:35,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:36,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:37,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:38,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:38,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:40:38,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:40:39,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:40,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:41,120 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:41,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:42,476 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:43,485 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:44,483 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:45,481 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:46,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:46,484 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:47,500 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:48,482 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:49,334 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:40:49,336 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:40:49,336 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:40:49,338 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:40:49,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:50,168 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:40:50,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:51,744 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:52,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:52,170 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:40:52,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:53,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:53,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:40:53,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:40:54,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:55,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:56,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:57,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:58,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:40:58,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:40:59,369 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:40:59,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:00,745 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:01,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:02,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:03,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:04,123 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:04,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:05,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:06,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:07,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:08,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:08,801 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:41:08,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:41:09,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:09,954 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:10,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:11,746 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:12,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:13,747 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:14,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:15,126 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:15,750 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:16,751 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:17,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:18,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:19,011 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:41:19,012 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:41:19,013 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:41:19,014 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:41:19,200 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:41:19,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:20,201 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:41:20,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:21,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:21,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:22,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:23,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:23,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:41:23,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:41:24,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:25,754 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:26,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:27,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:27,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:28,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:29,373 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:41:29,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:30,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:31,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:32,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:33,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:33,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:34,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:35,755 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:36,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:37,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:38,126 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:38,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:38,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:41:38,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:41:39,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:40,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:41,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:42,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:43,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:44,125 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:44,756 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:45,770 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:46,760 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:47,763 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:48,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:49,130 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:49,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:50,765 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:51,599 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:41:51,600 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:41:51,600 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:41:51,601 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:41:52,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:52,234 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:41:53,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:53,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:41:53,802 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:41:54,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:54,236 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:41:54,913 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:41:55,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:56,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:57,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:58,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:59,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:41:59,377 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:42:00,033 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:00,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:01,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:02,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:03,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:04,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:05,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:05,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:06,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:07,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:08,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:08,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:42:08,803 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:42:09,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:10,034 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:11,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:11,129 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:12,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:13,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:14,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:15,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:16,035 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:16,138 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:17,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:18,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:19,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:20,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:21,037 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:21,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:22,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:22,177 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:42:22,178 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:42:22,179 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:42:22,180 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:42:22,265 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:42:23,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:23,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:42:23,803 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:42:24,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:24,267 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:42:25,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:26,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:26,133 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:27,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:28,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:29,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:29,382 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:42:30,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:31,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:32,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:32,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:33,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:34,038 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:35,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:36,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:37,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:38,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:38,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:38,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:42:38,803 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:42:39,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:40,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:41,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:42,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:43,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:44,039 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:44,132 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:45,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:46,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:47,040 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:48,047 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:49,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:49,140 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:50,046 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:51,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:52,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:53,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:53,382 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:42:53,384 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:42:53,384 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:42:53,386 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:42:53,802 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:42:53,803 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:42:54,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:54,296 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:42:54,907 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:42:55,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:56,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:56,298 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:42:57,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:58,048 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:59,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:42:59,386 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:43:00,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:00,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:01,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:02,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:03,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:04,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:05,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:06,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:06,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:07,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:08,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:08,803 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:43:08,803 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:43:09,049 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:10,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:11,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:12,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:12,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:13,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:14,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:15,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:16,050 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:17,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:18,051 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:18,136 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:19,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:20,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:21,058 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:22,060 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:22,998 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: partial_history +2025-01-02 04:43:22,999 DEBUG SenderThread:2087140 [sender.py:send():379] send: history +2025-01-02 04:43:22,999 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: summary_record +2025-01-02 04:43:23,000 INFO SenderThread:2087140 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-02 04:43:23,335 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json +2025-01-02 04:43:23,406 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:23,804 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:43:23,805 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:43:23,921 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:24,142 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:24,336 INFO Thread-12 :2087140 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/files/output.log +2025-01-02 04:43:25,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:26,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:27,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:28,146 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:29,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:29,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:29,389 DEBUG SenderThread:2087140 [sender.py:send():379] send: stats +2025-01-02 04:43:30,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:31,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:32,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:33,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:34,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:35,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:35,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:36,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:37,147 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:38,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:38,804 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: stop_status +2025-01-02 04:43:38,804 DEBUG SenderThread:2087140 [sender.py:send_request():406] send_request: stop_status +2025-01-02 04:43:39,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:40,141 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: status_report +2025-01-02 04:43:40,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:41,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:42,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:43,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-02 04:43:44,148 DEBUG HandlerThread:2087140 [handler.py:handle_request():158] handle_request: internal_messages