Upload . with huggingface_hub

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+replay.mp4 filter=lfs diff=lfs merge=lfs -text

.summary/0/events.out.tfevents.1677745987.deee905a985a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:face7918fafbbb2d11b741679791d03e757587847c2cafcf91750660a64d0164
+size 14556

.summary/0/events.out.tfevents.1677746890.deee905a985a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81a557fdc7f5bc754df2041199a664db97809615334be1338e41c37a4ad08251
+size 1230

README.md

@@ -0,0 +1,56 @@
+---
+library_name: sample-factory
+tags:
+- deep-reinforcement-learning
+- reinforcement-learning
+- sample-factory
+model-index:
+- name: APPO
+  results:
+  - task:
+      type: reinforcement-learning
+      name: reinforcement-learning
+    dataset:
+      name: doom_health_gathering_supreme
+      type: doom_health_gathering_supreme
+    metrics:
+    - type: mean_reward
+      value: 3.94 +/- 0.66
+      name: mean_reward
+      verified: false
+---
+
+A(n) **APPO** model trained on the **doom_health_gathering_supreme** environment.
+
+This model was trained using Sample-Factory 2.0: https://github.com/alex-petrenko/sample-factory.
+Documentation for how to use Sample-Factory can be found at https://www.samplefactory.dev/
+
+
+## Downloading the model
+
+After installing Sample-Factory, download the model with:
+```
+python -m sample_factory.huggingface.load_from_hub -r nhiro3303/rl_course_vizdoom_health_gathering_supreme
+```
+
+    
+## Using the model
+
+To run the model after download, use the `enjoy` script corresponding to this environment:
+```
+python -m <path.to.enjoy.module> --algo=APPO --env=doom_health_gathering_supreme --train_dir=./train_dir --experiment=rl_course_vizdoom_health_gathering_supreme
+```
+
+
+You can also upload models to the Hugging Face Hub using the same script with the `--push_to_hub` flag.
+See https://www.samplefactory.dev/10-huggingface/huggingface/ for more details
+    
+## Training with this model
+
+To continue training with this model, use the `train` script corresponding to this environment:
+```
+python -m <path.to.train.module> --algo=APPO --env=doom_health_gathering_supreme --train_dir=./train_dir --experiment=rl_course_vizdoom_health_gathering_supreme --restart_behavior=resume --train_for_env_steps=10000000000
+```
+
+Note, you may have to adjust `--train_for_env_steps` to a suitably high number as the experiment will resume at the number of steps it concluded at.
+    

checkpoint_p0/checkpoint_000000000_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb2a7743d127850eaf55cd07c0fe9e6e853d1fd425fb3ebabc385147d8293997
+size 11938836

config.json

@@ -0,0 +1,142 @@
+{
+  "help": false,
+  "algo": "APPO",
+  "env": "doom_health_gathering_supreme",
+  "experiment": "default_experiment",
+  "train_dir": "/home/gpu/train_dir",
+  "restart_behavior": "resume",
+  "device": "gpu",
+  "seed": null,
+  "num_policies": 1,
+  "async_rl": true,
+  "serial_mode": false,
+  "batched_sampling": false,
+  "num_batches_to_accumulate": 2,
+  "worker_num_splits": 2,
+  "policy_workers_per_policy": 1,
+  "max_policy_lag": 1000,
+  "num_workers": 8,
+  "num_envs_per_worker": 4,
+  "batch_size": 1024,
+  "num_batches_per_epoch": 1,
+  "num_epochs": 1,
+  "rollout": 32,
+  "recurrence": 32,
+  "shuffle_minibatches": false,
+  "gamma": 0.99,
+  "reward_scale": 1.0,
+  "reward_clip": 1000.0,
+  "value_bootstrap": false,
+  "normalize_returns": true,
+  "exploration_loss_coeff": 0.001,
+  "value_loss_coeff": 0.5,
+  "kl_loss_coeff": 0.0,
+  "exploration_loss": "symmetric_kl",
+  "gae_lambda": 0.95,
+  "ppo_clip_ratio": 0.1,
+  "ppo_clip_value": 0.2,
+  "with_vtrace": false,
+  "vtrace_rho": 1.0,
+  "vtrace_c": 1.0,
+  "optimizer": "adam",
+  "adam_eps": 1e-06,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "max_grad_norm": 4.0,
+  "learning_rate": 0.0001,
+  "lr_schedule": "constant",
+  "lr_schedule_kl_threshold": 0.008,
+  "lr_adaptive_min": 1e-06,
+  "lr_adaptive_max": 0.01,
+  "obs_subtract_mean": 0.0,
+  "obs_scale": 255.0,
+  "normalize_input": true,
+  "normalize_input_keys": null,
+  "decorrelate_experience_max_seconds": 0,
+  "decorrelate_envs_on_one_worker": true,
+  "actor_worker_gpus": [],
+  "set_workers_cpu_affinity": true,
+  "force_envs_single_thread": false,
+  "default_niceness": 0,
+  "log_to_file": true,
+  "experiment_summaries_interval": 10,
+  "flush_summaries_interval": 30,
+  "stats_avg": 100,
+  "summaries_use_frameskip": true,
+  "heartbeat_interval": 20,
+  "heartbeat_reporting_interval": 600,
+  "train_for_env_steps": 4000000,
+  "train_for_seconds": 10000000000,
+  "save_every_sec": 120,
+  "keep_checkpoints": 2,
+  "load_checkpoint_kind": "latest",
+  "save_milestones_sec": -1,
+  "save_best_every_sec": 5,
+  "save_best_metric": "reward",
+  "save_best_after": 100000,
+  "benchmark": false,
+  "encoder_mlp_layers": [
+    512,
+    512
+  ],
+  "encoder_conv_architecture": "convnet_simple",
+  "encoder_conv_mlp_layers": [
+    512
+  ],
+  "use_rnn": true,
+  "rnn_size": 512,
+  "rnn_type": "gru",
+  "rnn_num_layers": 1,
+  "decoder_mlp_layers": [],
+  "nonlinearity": "elu",
+  "policy_initialization": "orthogonal",
+  "policy_init_gain": 1.0,
+  "actor_critic_share_weights": true,
+  "adaptive_stddev": true,
+  "continuous_tanh_scale": 0.0,
+  "initial_stddev": 1.0,
+  "use_env_info_cache": false,
+  "env_gpu_actions": false,
+  "env_gpu_observations": true,
+  "env_frameskip": 4,
+  "env_framestack": 1,
+  "pixel_format": "CHW",
+  "use_record_episode_statistics": false,
+  "with_wandb": false,
+  "wandb_user": null,
+  "wandb_project": "sample_factory",
+  "wandb_group": null,
+  "wandb_job_type": "SF",
+  "wandb_tags": [],
+  "with_pbt": false,
+  "pbt_mix_policies_in_one_env": true,
+  "pbt_period_env_steps": 5000000,
+  "pbt_start_mutation": 20000000,
+  "pbt_replace_fraction": 0.3,
+  "pbt_mutation_rate": 0.15,
+  "pbt_replace_reward_gap": 0.1,
+  "pbt_replace_reward_gap_absolute": 1e-06,
+  "pbt_optimize_gamma": false,
+  "pbt_target_objective": "true_objective",
+  "pbt_perturb_min": 1.1,
+  "pbt_perturb_max": 1.5,
+  "num_agents": -1,
+  "num_humans": 0,
+  "num_bots": -1,
+  "start_bot_difficulty": null,
+  "timelimit": null,
+  "res_w": 128,
+  "res_h": 72,
+  "wide_aspect_ratio": false,
+  "eval_env_frameskip": 1,
+  "fps": 35,
+  "command_line": "--env=doom_health_gathering_supreme --num_workers=8 --num_envs_per_worker=4 --train_for_env_steps=4000000",
+  "cli_args": {
+    "env": "doom_health_gathering_supreme",
+    "num_workers": 8,
+    "num_envs_per_worker": 4,
+    "train_for_env_steps": 4000000
+  },
+  "git_hash": "unknown",
+  "git_repo_name": "not a git repository"
+}

replay.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d5850b57a33e86bced6b3fd1294fceb3faec99c197b47d416c2779fa79c7ab
+size 5827554

sf_log.txt

@@ -0,0 +1,704 @@
+[2023-03-02 08:33:09,561][08277] Saving configuration to /home/gpu/train_dir/default_experiment/config.json...
+[2023-03-02 08:33:09,561][08277] Rollout worker 0 uses device cpu
+[2023-03-02 08:33:09,561][08277] Rollout worker 1 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 2 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 3 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 4 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 5 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 6 uses device cpu
+[2023-03-02 08:33:09,562][08277] Rollout worker 7 uses device cpu
+[2023-03-02 08:33:09,608][08277] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:33:09,608][08277] InferenceWorker_p0-w0: min num requests: 2
+[2023-03-02 08:33:09,624][08277] Starting all processes...
+[2023-03-02 08:33:09,625][08277] Starting process learner_proc0
+[2023-03-02 08:33:10,271][08277] Starting all processes...
+[2023-03-02 08:33:10,275][08335] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:33:10,275][08335] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0
+[2023-03-02 08:33:10,275][08277] Starting process inference_proc0-0
+[2023-03-02 08:33:10,275][08277] Starting process rollout_proc0
+[2023-03-02 08:33:10,276][08277] Starting process rollout_proc1
+[2023-03-02 08:33:10,276][08277] Starting process rollout_proc2
+[2023-03-02 08:33:10,279][08277] Starting process rollout_proc3
+[2023-03-02 08:33:10,279][08277] Starting process rollout_proc4
+[2023-03-02 08:33:10,280][08277] Starting process rollout_proc5
+[2023-03-02 08:33:10,280][08277] Starting process rollout_proc6
+[2023-03-02 08:33:10,280][08277] Starting process rollout_proc7
+[2023-03-02 08:33:10,310][08335] Num visible devices: 1
+[2023-03-02 08:33:10,353][08335] Starting seed is not provided
+[2023-03-02 08:33:10,353][08335] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:33:10,353][08335] Initializing actor-critic model on device cuda:0
+[2023-03-02 08:33:10,354][08335] RunningMeanStd input shape: (3, 72, 128)
+[2023-03-02 08:33:10,355][08335] RunningMeanStd input shape: (1,)
+[2023-03-02 08:33:10,363][08335] ConvEncoder: input_channels=3
+[2023-03-02 08:33:10,681][08335] Conv encoder output size: 512
+[2023-03-02 08:33:10,681][08335] Policy head output size: 512
+[2023-03-02 08:33:10,732][08335] Created Actor Critic model with architecture:
+[2023-03-02 08:33:10,733][08335] ActorCriticSharedWeights(
+  (obs_normalizer): ObservationNormalizer(
+    (running_mean_std): RunningMeanStdDictInPlace(
+      (running_mean_std): ModuleDict(
+        (obs): RunningMeanStdInPlace()
+      )
+    )
+  )
+  (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace)
+  (encoder): VizdoomEncoder(
+    (basic_encoder): ConvEncoder(
+      (enc): RecursiveScriptModule(
+        original_name=ConvEncoderImpl
+        (conv_head): RecursiveScriptModule(
+          original_name=Sequential
+          (0): RecursiveScriptModule(original_name=Conv2d)
+          (1): RecursiveScriptModule(original_name=ELU)
+          (2): RecursiveScriptModule(original_name=Conv2d)
+          (3): RecursiveScriptModule(original_name=ELU)
+          (4): RecursiveScriptModule(original_name=Conv2d)
+          (5): RecursiveScriptModule(original_name=ELU)
+        )
+        (mlp_layers): RecursiveScriptModule(
+          original_name=Sequential
+          (0): RecursiveScriptModule(original_name=Linear)
+          (1): RecursiveScriptModule(original_name=ELU)
+        )
+      )
+    )
+  )
+  (core): ModelCoreRNN(
+    (core): GRU(512, 512)
+  )
+  (decoder): MlpDecoder(
+    (mlp): Identity()
+  )
+  (critic_linear): Linear(in_features=512, out_features=1, bias=True)
+  (action_parameterization): ActionParameterizationDefault(
+    (distribution_linear): Linear(in_features=512, out_features=5, bias=True)
+  )
+)
+[2023-03-02 08:33:11,319][08369] Worker 6 uses CPU cores [12, 13]
+[2023-03-02 08:33:11,329][08365] Worker 1 uses CPU cores [2, 3]
+[2023-03-02 08:33:11,355][08366] Worker 3 uses CPU cores [6, 7]
+[2023-03-02 08:33:11,366][08363] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:33:11,366][08363] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0
+[2023-03-02 08:33:11,378][08368] Worker 5 uses CPU cores [10, 11]
+[2023-03-02 08:33:11,379][08363] Num visible devices: 1
+[2023-03-02 08:33:11,410][08364] Worker 0 uses CPU cores [0, 1]
+[2023-03-02 08:33:11,420][08367] Worker 2 uses CPU cores [4, 5]
+[2023-03-02 08:33:11,425][08370] Worker 7 uses CPU cores [14, 15]
+[2023-03-02 08:33:11,426][08371] Worker 4 uses CPU cores [8, 9]
+[2023-03-02 08:33:14,220][08335] Using optimizer <class 'torch.optim.adam.Adam'>
+[2023-03-02 08:33:14,221][08335] No checkpoints found
+[2023-03-02 08:33:14,221][08335] Did not load from checkpoint, starting from scratch!
+[2023-03-02 08:33:14,221][08335] Initialized policy 0 weights for model version 0
+[2023-03-02 08:33:14,225][08335] LearnerWorker_p0 finished initialization!
+[2023-03-02 08:33:14,225][08335] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:33:14,336][08363] Unhandled exception CUDA error: invalid resource handle
+CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
+For debugging consider passing CUDA_LAUNCH_BLOCKING=1. in evt loop inference_proc0-0_evt_loop
+[2023-03-02 08:33:17,898][08277] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:29,604][08277] Heartbeat connected on Batcher_0
+[2023-03-02 08:33:29,606][08277] Heartbeat connected on LearnerWorker_p0
+[2023-03-02 08:33:29,611][08277] Heartbeat connected on RolloutWorker_w0
+[2023-03-02 08:33:29,612][08277] Heartbeat connected on RolloutWorker_w1
+[2023-03-02 08:33:29,614][08277] Heartbeat connected on RolloutWorker_w2
+[2023-03-02 08:33:29,616][08277] Heartbeat connected on RolloutWorker_w3
+[2023-03-02 08:33:29,618][08277] Heartbeat connected on RolloutWorker_w4
+[2023-03-02 08:33:29,620][08277] Heartbeat connected on RolloutWorker_w5
+[2023-03-02 08:33:29,622][08277] Heartbeat connected on RolloutWorker_w6
+[2023-03-02 08:33:29,624][08277] Heartbeat connected on RolloutWorker_w7
+[2023-03-02 08:33:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:33:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:34:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:35:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:35:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:36:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:37:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:37:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:38:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:39:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:39:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:40:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:41:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:41:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:42:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:43:07,900][08277] Components not started: InferenceWorker_p0-w0, wait_time=600.0 seconds
+[2023-03-02 08:43:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:43:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:44:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:45:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:45:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:32,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:37,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:42,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:47,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:52,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:46:57,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:02,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:07,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:07,899][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:47:12,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:17,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:22,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:27,898][08277] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:47:31,424][08277] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 8277], exiting...
+[2023-03-02 08:47:31,424][08371] Stopping RolloutWorker_w4...
+[2023-03-02 08:47:31,424][08365] Stopping RolloutWorker_w1...
+[2023-03-02 08:47:31,424][08368] Stopping RolloutWorker_w5...
+[2023-03-02 08:47:31,424][08364] Stopping RolloutWorker_w0...
+[2023-03-02 08:47:31,424][08277] Runner profile tree view:
+main_loop: 861.8000
+[2023-03-02 08:47:31,424][08367] Stopping RolloutWorker_w2...
+[2023-03-02 08:47:31,425][08277] Collected {0: 0}, FPS: 0.0
+[2023-03-02 08:47:31,424][08369] Stopping RolloutWorker_w6...
+[2023-03-02 08:47:31,424][08366] Stopping RolloutWorker_w3...
+[2023-03-02 08:47:31,425][08365] Loop rollout_proc1_evt_loop terminating...
+[2023-03-02 08:47:31,425][08371] Loop rollout_proc4_evt_loop terminating...
+[2023-03-02 08:47:31,425][08368] Loop rollout_proc5_evt_loop terminating...
+[2023-03-02 08:47:31,425][08335] Stopping Batcher_0...
+[2023-03-02 08:47:31,425][08370] Stopping RolloutWorker_w7...
+[2023-03-02 08:47:31,425][08364] Loop rollout_proc0_evt_loop terminating...
+[2023-03-02 08:47:31,425][08367] Loop rollout_proc2_evt_loop terminating...
+[2023-03-02 08:47:31,425][08369] Loop rollout_proc6_evt_loop terminating...
+[2023-03-02 08:47:31,425][08366] Loop rollout_proc3_evt_loop terminating...
+[2023-03-02 08:47:31,425][08370] Loop rollout_proc7_evt_loop terminating...
+[2023-03-02 08:47:31,425][08335] Loop batcher_evt_loop terminating...
+[2023-03-02 08:47:31,426][08335] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:47:31,446][08335] Stopping LearnerWorker_p0...
+[2023-03-02 08:47:31,447][08335] Loop learner_proc0_evt_loop terminating...
+[2023-03-02 08:47:31,510][08277] Loading existing experiment configuration from /home/gpu/train_dir/default_experiment/config.json
+[2023-03-02 08:47:31,510][08277] Overriding arg 'num_workers' with value 1 passed from command line
+[2023-03-02 08:47:31,511][08277] Adding new argument 'no_render'=True that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'save_video'=True that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'video_name'=None that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'max_num_episodes'=10 that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'push_to_hub'=False that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'hf_repository'=None that is not in the saved config file!
+[2023-03-02 08:47:31,511][08277] Adding new argument 'policy_index'=0 that is not in the saved config file!
+[2023-03-02 08:47:31,512][08277] Adding new argument 'eval_deterministic'=False that is not in the saved config file!
+[2023-03-02 08:47:31,512][08277] Adding new argument 'train_script'=None that is not in the saved config file!
+[2023-03-02 08:47:31,512][08277] Adding new argument 'enjoy_script'=None that is not in the saved config file!
+[2023-03-02 08:47:31,512][08277] Using frameskip 1 and render_action_repeat=4 for evaluation
+[2023-03-02 08:47:31,521][08277] Doom resolution: 160x120, resize resolution: (128, 72)
+[2023-03-02 08:47:31,521][08277] RunningMeanStd input shape: (3, 72, 128)
+[2023-03-02 08:47:31,522][08277] RunningMeanStd input shape: (1,)
+[2023-03-02 08:47:31,533][08277] ConvEncoder: input_channels=3
+[2023-03-02 08:47:31,775][08277] Conv encoder output size: 512
+[2023-03-02 08:47:31,776][08277] Policy head output size: 512
+[2023-03-02 08:47:33,379][08277] Loading state from checkpoint /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:47:34,725][08277] Num frames 100...
+[2023-03-02 08:47:34,819][08277] Num frames 200...
+[2023-03-02 08:47:34,913][08277] Num frames 300...
+[2023-03-02 08:47:35,007][08277] Num frames 400...
+[2023-03-02 08:47:35,136][08277] Avg episode rewards: #0: 6.800, true rewards: #0: 4.800
+[2023-03-02 08:47:35,136][08277] Avg episode reward: 6.800, avg true_objective: 4.800
+[2023-03-02 08:47:35,155][08277] Num frames 500...
+[2023-03-02 08:47:35,248][08277] Num frames 600...
+[2023-03-02 08:47:35,342][08277] Num frames 700...
+[2023-03-02 08:47:35,438][08277] Num frames 800...
+[2023-03-02 08:47:35,551][08277] Avg episode rewards: #0: 5.320, true rewards: #0: 4.320
+[2023-03-02 08:47:35,552][08277] Avg episode reward: 5.320, avg true_objective: 4.320
+[2023-03-02 08:47:35,590][08277] Num frames 900...
+[2023-03-02 08:47:35,693][08277] Num frames 1000...
+[2023-03-02 08:47:35,790][08277] Num frames 1100...
+[2023-03-02 08:47:35,885][08277] Num frames 1200...
+[2023-03-02 08:47:35,984][08277] Avg episode rewards: #0: 4.827, true rewards: #0: 4.160
+[2023-03-02 08:47:35,984][08277] Avg episode reward: 4.827, avg true_objective: 4.160
+[2023-03-02 08:47:36,035][08277] Num frames 1300...
+[2023-03-02 08:47:36,131][08277] Num frames 1400...
+[2023-03-02 08:47:36,226][08277] Num frames 1500...
+[2023-03-02 08:47:36,321][08277] Num frames 1600...
+[2023-03-02 08:47:36,404][08277] Avg episode rewards: #0: 4.580, true rewards: #0: 4.080
+[2023-03-02 08:47:36,405][08277] Avg episode reward: 4.580, avg true_objective: 4.080
+[2023-03-02 08:47:36,469][08277] Num frames 1700...
+[2023-03-02 08:47:36,563][08277] Num frames 1800...
+[2023-03-02 08:47:36,657][08277] Num frames 1900...
+[2023-03-02 08:47:36,751][08277] Num frames 2000...
+[2023-03-02 08:47:36,821][08277] Avg episode rewards: #0: 4.432, true rewards: #0: 4.032
+[2023-03-02 08:47:36,821][08277] Avg episode reward: 4.432, avg true_objective: 4.032
+[2023-03-02 08:47:36,905][08277] Num frames 2100...
+[2023-03-02 08:47:36,999][08277] Num frames 2200...
+[2023-03-02 08:47:37,093][08277] Num frames 2300...
+[2023-03-02 08:47:37,187][08277] Num frames 2400...
+[2023-03-02 08:47:37,300][08277] Avg episode rewards: #0: 4.607, true rewards: #0: 4.107
+[2023-03-02 08:47:37,301][08277] Avg episode reward: 4.607, avg true_objective: 4.107
+[2023-03-02 08:47:37,335][08277] Num frames 2500...
+[2023-03-02 08:47:37,431][08277] Num frames 2600...
+[2023-03-02 08:47:37,525][08277] Num frames 2700...
+[2023-03-02 08:47:37,620][08277] Num frames 2800...
+[2023-03-02 08:47:37,718][08277] Avg episode rewards: #0: 4.497, true rewards: #0: 4.069
+[2023-03-02 08:47:37,718][08277] Avg episode reward: 4.497, avg true_objective: 4.069
+[2023-03-02 08:47:37,768][08277] Num frames 2900...
+[2023-03-02 08:47:37,862][08277] Num frames 3000...
+[2023-03-02 08:47:37,956][08277] Num frames 3100...
+[2023-03-02 08:47:38,051][08277] Num frames 3200...
+[2023-03-02 08:47:38,134][08277] Avg episode rewards: #0: 4.415, true rewards: #0: 4.040
+[2023-03-02 08:47:38,134][08277] Avg episode reward: 4.415, avg true_objective: 4.040
+[2023-03-02 08:47:38,199][08277] Num frames 3300...
+[2023-03-02 08:47:38,294][08277] Num frames 3400...
+[2023-03-02 08:47:38,395][08277] Num frames 3500...
+[2023-03-02 08:47:38,502][08277] Num frames 3600...
+[2023-03-02 08:47:38,571][08277] Avg episode rewards: #0: 4.351, true rewards: #0: 4.018
+[2023-03-02 08:47:38,571][08277] Avg episode reward: 4.351, avg true_objective: 4.018
+[2023-03-02 08:47:38,658][08277] Num frames 3700...
+[2023-03-02 08:47:38,760][08277] Num frames 3800...
+[2023-03-02 08:47:38,858][08277] Num frames 3900...
+[2023-03-02 08:47:38,962][08277] Num frames 4000...
+[2023-03-02 08:47:39,012][08277] Avg episode rewards: #0: 4.300, true rewards: #0: 4.000
+[2023-03-02 08:47:39,013][08277] Avg episode reward: 4.300, avg true_objective: 4.000
+[2023-03-02 08:47:42,935][08277] Replay video saved to /home/gpu/train_dir/default_experiment/replay.mp4!
+[2023-03-02 08:48:11,637][08718] Saving configuration to /home/gpu/train_dir/default_experiment/config.json...
+[2023-03-02 08:48:11,638][08718] Rollout worker 0 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 1 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 2 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 3 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 4 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 5 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 6 uses device cpu
+[2023-03-02 08:48:11,638][08718] Rollout worker 7 uses device cpu
+[2023-03-02 08:48:11,665][08718] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:48:11,666][08718] InferenceWorker_p0-w0: min num requests: 2
+[2023-03-02 08:48:11,682][08718] Starting all processes...
+[2023-03-02 08:48:11,682][08718] Starting process learner_proc0
+[2023-03-02 08:48:12,341][08718] Starting all processes...
+[2023-03-02 08:48:12,344][08718] Starting process inference_proc0-0
+[2023-03-02 08:48:12,344][08776] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:48:12,344][08718] Starting process rollout_proc0
+[2023-03-02 08:48:12,344][08776] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0
+[2023-03-02 08:48:12,344][08718] Starting process rollout_proc1
+[2023-03-02 08:48:12,344][08718] Starting process rollout_proc2
+[2023-03-02 08:48:12,344][08718] Starting process rollout_proc3
+[2023-03-02 08:48:12,346][08718] Starting process rollout_proc4
+[2023-03-02 08:48:12,346][08718] Starting process rollout_proc5
+[2023-03-02 08:48:12,347][08718] Starting process rollout_proc6
+[2023-03-02 08:48:12,348][08718] Starting process rollout_proc7
+[2023-03-02 08:48:12,379][08776] Num visible devices: 1
+[2023-03-02 08:48:12,406][08776] Starting seed is not provided
+[2023-03-02 08:48:12,407][08776] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:48:12,407][08776] Initializing actor-critic model on device cuda:0
+[2023-03-02 08:48:12,407][08776] RunningMeanStd input shape: (3, 72, 128)
+[2023-03-02 08:48:12,408][08776] RunningMeanStd input shape: (1,)
+[2023-03-02 08:48:12,418][08776] ConvEncoder: input_channels=3
+[2023-03-02 08:48:12,624][08776] Conv encoder output size: 512
+[2023-03-02 08:48:12,625][08776] Policy head output size: 512
+[2023-03-02 08:48:12,637][08776] Created Actor Critic model with architecture:
+[2023-03-02 08:48:12,638][08776] ActorCriticSharedWeights(
+  (obs_normalizer): ObservationNormalizer(
+    (running_mean_std): RunningMeanStdDictInPlace(
+      (running_mean_std): ModuleDict(
+        (obs): RunningMeanStdInPlace()
+      )
+    )
+  )
+  (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace)
+  (encoder): VizdoomEncoder(
+    (basic_encoder): ConvEncoder(
+      (enc): RecursiveScriptModule(
+        original_name=ConvEncoderImpl
+        (conv_head): RecursiveScriptModule(
+          original_name=Sequential
+          (0): RecursiveScriptModule(original_name=Conv2d)
+          (1): RecursiveScriptModule(original_name=ELU)
+          (2): RecursiveScriptModule(original_name=Conv2d)
+          (3): RecursiveScriptModule(original_name=ELU)
+          (4): RecursiveScriptModule(original_name=Conv2d)
+          (5): RecursiveScriptModule(original_name=ELU)
+        )
+        (mlp_layers): RecursiveScriptModule(
+          original_name=Sequential
+          (0): RecursiveScriptModule(original_name=Linear)
+          (1): RecursiveScriptModule(original_name=ELU)
+        )
+      )
+    )
+  )
+  (core): ModelCoreRNN(
+    (core): GRU(512, 512)
+  )
+  (decoder): MlpDecoder(
+    (mlp): Identity()
+  )
+  (critic_linear): Linear(in_features=512, out_features=1, bias=True)
+  (action_parameterization): ActionParameterizationDefault(
+    (distribution_linear): Linear(in_features=512, out_features=5, bias=True)
+  )
+)
+[2023-03-02 08:48:13,355][08812] Worker 7 uses CPU cores [14, 15]
+[2023-03-02 08:48:13,396][08810] Worker 5 uses CPU cores [10, 11]
+[2023-03-02 08:48:13,399][08811] Worker 6 uses CPU cores [12, 13]
+[2023-03-02 08:48:13,430][08805] Worker 0 uses CPU cores [0, 1]
+[2023-03-02 08:48:13,433][08808] Worker 3 uses CPU cores [6, 7]
+[2023-03-02 08:48:13,439][08807] Worker 2 uses CPU cores [4, 5]
+[2023-03-02 08:48:13,482][08809] Worker 4 uses CPU cores [8, 9]
+[2023-03-02 08:48:13,485][08804] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:48:13,485][08804] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0
+[2023-03-02 08:48:13,486][08806] Worker 1 uses CPU cores [2, 3]
+[2023-03-02 08:48:13,500][08804] Num visible devices: 1
+[2023-03-02 08:48:14,689][08776] Using optimizer <class 'torch.optim.adam.Adam'>
+[2023-03-02 08:48:14,689][08776] Loading state from checkpoint /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:48:14,696][08776] Loading model from checkpoint
+[2023-03-02 08:48:14,697][08776] Loaded experiment state at self.train_step=0, self.env_steps=0
+[2023-03-02 08:48:14,697][08776] Initialized policy 0 weights for model version 0
+[2023-03-02 08:48:14,700][08776] LearnerWorker_p0 finished initialization!
+[2023-03-02 08:48:14,700][08776] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-03-02 08:48:14,800][08804] Unhandled exception CUDA error: invalid resource handle
+CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
+For debugging consider passing CUDA_LAUNCH_BLOCKING=1. in evt loop inference_proc0-0_evt_loop
+[2023-03-02 08:48:15,123][08718] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:20,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:25,122][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:30,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:31,661][08718] Heartbeat connected on Batcher_0
+[2023-03-02 08:48:31,663][08718] Heartbeat connected on LearnerWorker_p0
+[2023-03-02 08:48:31,668][08718] Heartbeat connected on RolloutWorker_w0
+[2023-03-02 08:48:31,670][08718] Heartbeat connected on RolloutWorker_w1
+[2023-03-02 08:48:31,672][08718] Heartbeat connected on RolloutWorker_w2
+[2023-03-02 08:48:31,674][08718] Heartbeat connected on RolloutWorker_w3
+[2023-03-02 08:48:31,676][08718] Heartbeat connected on RolloutWorker_w4
+[2023-03-02 08:48:31,678][08718] Heartbeat connected on RolloutWorker_w5
+[2023-03-02 08:48:31,679][08718] Heartbeat connected on RolloutWorker_w6
+[2023-03-02 08:48:31,681][08718] Heartbeat connected on RolloutWorker_w7
+[2023-03-02 08:48:35,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:40,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:45,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:50,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:48:55,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:00,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:05,122][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:10,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:15,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:20,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:25,123][08718] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-03-02 08:49:27,382][08718] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 8718], exiting...
+[2023-03-02 08:49:27,382][08718] Runner profile tree view:
+main_loop: 75.7006
+[2023-03-02 08:49:27,382][08809] Stopping RolloutWorker_w4...
+[2023-03-02 08:49:27,382][08810] Stopping RolloutWorker_w5...
+[2023-03-02 08:49:27,382][08718] Collected {0: 0}, FPS: 0.0
+[2023-03-02 08:49:27,382][08808] Stopping RolloutWorker_w3...
+[2023-03-02 08:49:27,382][08811] Stopping RolloutWorker_w6...
+[2023-03-02 08:49:27,382][08809] Loop rollout_proc4_evt_loop terminating...
+[2023-03-02 08:49:27,382][08810] Loop rollout_proc5_evt_loop terminating...
+[2023-03-02 08:49:27,382][08776] Stopping Batcher_0...
+[2023-03-02 08:49:27,382][08812] Stopping RolloutWorker_w7...
+[2023-03-02 08:49:27,383][08808] Loop rollout_proc3_evt_loop terminating...
+[2023-03-02 08:49:27,383][08811] Loop rollout_proc6_evt_loop terminating...
+[2023-03-02 08:49:27,383][08807] Stopping RolloutWorker_w2...
+[2023-03-02 08:49:27,383][08812] Loop rollout_proc7_evt_loop terminating...
+[2023-03-02 08:49:27,383][08806] Stopping RolloutWorker_w1...
+[2023-03-02 08:49:27,383][08776] Loop batcher_evt_loop terminating...
+[2023-03-02 08:49:27,383][08805] Stopping RolloutWorker_w0...
+[2023-03-02 08:49:27,383][08807] Loop rollout_proc2_evt_loop terminating...
+[2023-03-02 08:49:27,383][08806] Loop rollout_proc1_evt_loop terminating...
+[2023-03-02 08:49:27,383][08805] Loop rollout_proc0_evt_loop terminating...
+[2023-03-02 08:49:27,384][08776] Saving /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:49:27,405][08776] Stopping LearnerWorker_p0...
+[2023-03-02 08:49:27,405][08776] Loop learner_proc0_evt_loop terminating...
+[2023-03-02 08:49:27,470][08718] Loading existing experiment configuration from /home/gpu/train_dir/default_experiment/config.json
+[2023-03-02 08:49:27,471][08718] Overriding arg 'num_workers' with value 1 passed from command line
+[2023-03-02 08:49:27,471][08718] Adding new argument 'no_render'=True that is not in the saved config file!
+[2023-03-02 08:49:27,471][08718] Adding new argument 'save_video'=True that is not in the saved config file!
+[2023-03-02 08:49:27,471][08718] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'video_name'=None that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'max_num_episodes'=10 that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'push_to_hub'=False that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'hf_repository'=None that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'policy_index'=0 that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'eval_deterministic'=False that is not in the saved config file!
+[2023-03-02 08:49:27,472][08718] Adding new argument 'train_script'=None that is not in the saved config file!
+[2023-03-02 08:49:27,473][08718] Adding new argument 'enjoy_script'=None that is not in the saved config file!
+[2023-03-02 08:49:27,473][08718] Using frameskip 1 and render_action_repeat=4 for evaluation
+[2023-03-02 08:49:27,484][08718] Doom resolution: 160x120, resize resolution: (128, 72)
+[2023-03-02 08:49:27,485][08718] RunningMeanStd input shape: (3, 72, 128)
+[2023-03-02 08:49:27,486][08718] RunningMeanStd input shape: (1,)
+[2023-03-02 08:49:27,496][08718] ConvEncoder: input_channels=3
+[2023-03-02 08:49:27,714][08718] Conv encoder output size: 512
+[2023-03-02 08:49:27,714][08718] Policy head output size: 512
+[2023-03-02 08:49:29,351][08718] Loading state from checkpoint /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:49:30,044][08718] Num frames 100...
+[2023-03-02 08:49:30,139][08718] Num frames 200...
+[2023-03-02 08:49:30,233][08718] Num frames 300...
+[2023-03-02 08:49:30,327][08718] Num frames 400...
+[2023-03-02 08:49:30,426][08718] Avg episode rewards: #0: 5.480, true rewards: #0: 4.480
+[2023-03-02 08:49:30,426][08718] Avg episode reward: 5.480, avg true_objective: 4.480
+[2023-03-02 08:49:30,477][08718] Num frames 500...
+[2023-03-02 08:49:30,577][08718] Num frames 600...
+[2023-03-02 08:49:30,678][08718] Num frames 700...
+[2023-03-02 08:49:30,780][08718] Num frames 800...
+[2023-03-02 08:49:30,863][08718] Avg episode rewards: #0: 4.660, true rewards: #0: 4.160
+[2023-03-02 08:49:30,863][08718] Avg episode reward: 4.660, avg true_objective: 4.160
+[2023-03-02 08:49:30,929][08718] Num frames 900...
+[2023-03-02 08:49:31,024][08718] Num frames 1000...
+[2023-03-02 08:49:31,118][08718] Num frames 1100...
+[2023-03-02 08:49:31,213][08718] Num frames 1200...
+[2023-03-02 08:49:31,281][08718] Avg episode rewards: #0: 4.387, true rewards: #0: 4.053
+[2023-03-02 08:49:31,281][08718] Avg episode reward: 4.387, avg true_objective: 4.053
+[2023-03-02 08:49:31,361][08718] Num frames 1300...
+[2023-03-02 08:49:31,458][08718] Num frames 1400...
+[2023-03-02 08:49:31,554][08718] Num frames 1500...
+[2023-03-02 08:49:31,650][08718] Num frames 1600...
+[2023-03-02 08:49:31,744][08718] Num frames 1700...
+[2023-03-02 08:49:31,858][08718] Avg episode rewards: #0: 5.150, true rewards: #0: 4.400
+[2023-03-02 08:49:31,858][08718] Avg episode reward: 5.150, avg true_objective: 4.400
+[2023-03-02 08:49:31,898][08718] Num frames 1800...
+[2023-03-02 08:49:31,993][08718] Num frames 1900...
+[2023-03-02 08:49:32,089][08718] Num frames 2000...
+[2023-03-02 08:49:32,184][08718] Num frames 2100...
+[2023-03-02 08:49:32,308][08718] Avg episode rewards: #0: 5.152, true rewards: #0: 4.352
+[2023-03-02 08:49:32,308][08718] Avg episode reward: 5.152, avg true_objective: 4.352
+[2023-03-02 08:49:32,331][08718] Num frames 2200...
+[2023-03-02 08:49:32,427][08718] Num frames 2300...
+[2023-03-02 08:49:32,523][08718] Num frames 2400...
+[2023-03-02 08:49:32,616][08718] Num frames 2500...
+[2023-03-02 08:49:32,727][08718] Avg episode rewards: #0: 4.933, true rewards: #0: 4.267
+[2023-03-02 08:49:32,727][08718] Avg episode reward: 4.933, avg true_objective: 4.267
+[2023-03-02 08:49:32,765][08718] Num frames 2600...
+[2023-03-02 08:49:32,860][08718] Num frames 2700...
+[2023-03-02 08:49:32,955][08718] Num frames 2800...
+[2023-03-02 08:49:33,049][08718] Num frames 2900...
+[2023-03-02 08:49:33,144][08718] Avg episode rewards: #0: 4.777, true rewards: #0: 4.206
+[2023-03-02 08:49:33,145][08718] Avg episode reward: 4.777, avg true_objective: 4.206
+[2023-03-02 08:49:33,200][08718] Num frames 3000...
+[2023-03-02 08:49:33,295][08718] Num frames 3100...
+[2023-03-02 08:49:33,391][08718] Num frames 3200...
+[2023-03-02 08:49:33,442][08718] Avg episode rewards: #0: 4.500, true rewards: #0: 4.000
+[2023-03-02 08:49:33,442][08718] Avg episode reward: 4.500, avg true_objective: 4.000
+[2023-03-02 08:49:33,539][08718] Num frames 3300...
+[2023-03-02 08:49:33,635][08718] Num frames 3400...
+[2023-03-02 08:49:33,729][08718] Num frames 3500...
+[2023-03-02 08:49:33,824][08718] Num frames 3600...
+[2023-03-02 08:49:33,921][08718] Avg episode rewards: #0: 4.609, true rewards: #0: 4.053
+[2023-03-02 08:49:33,922][08718] Avg episode reward: 4.609, avg true_objective: 4.053
+[2023-03-02 08:49:33,972][08718] Num frames 3700...
+[2023-03-02 08:49:34,065][08718] Num frames 3800...
+[2023-03-02 08:49:34,160][08718] Num frames 3900...
+[2023-03-02 08:49:34,286][08718] Num frames 4000...
+[2023-03-02 08:49:34,375][08718] Avg episode rewards: #0: 4.532, true rewards: #0: 4.032
+[2023-03-02 08:49:34,376][08718] Avg episode reward: 4.532, avg true_objective: 4.032
+[2023-03-02 08:49:38,275][08718] Replay video saved to /home/gpu/train_dir/default_experiment/replay.mp4!
+[2023-03-02 08:50:05,350][08718] Loading existing experiment configuration from /home/gpu/train_dir/default_experiment/config.json
+[2023-03-02 08:50:05,350][08718] Overriding arg 'num_workers' with value 1 passed from command line
+[2023-03-02 08:50:05,350][08718] Adding new argument 'no_render'=True that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'save_video'=True that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'video_name'=None that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'max_num_frames'=100000 that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'max_num_episodes'=10 that is not in the saved config file!
+[2023-03-02 08:50:05,350][08718] Adding new argument 'push_to_hub'=True that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Adding new argument 'hf_repository'='nhiro3303/rl_course_vizdoom_health_gathering_supreme' that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Adding new argument 'policy_index'=0 that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Adding new argument 'eval_deterministic'=False that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Adding new argument 'train_script'=None that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Adding new argument 'enjoy_script'=None that is not in the saved config file!
+[2023-03-02 08:50:05,351][08718] Using frameskip 1 and render_action_repeat=4 for evaluation
+[2023-03-02 08:50:05,354][08718] RunningMeanStd input shape: (3, 72, 128)
+[2023-03-02 08:50:05,354][08718] RunningMeanStd input shape: (1,)
+[2023-03-02 08:50:05,361][08718] ConvEncoder: input_channels=3
+[2023-03-02 08:50:05,386][08718] Conv encoder output size: 512
+[2023-03-02 08:50:05,387][08718] Policy head output size: 512
+[2023-03-02 08:50:05,412][08718] Loading state from checkpoint /home/gpu/train_dir/default_experiment/checkpoint_p0/checkpoint_000000000_0.pth...
+[2023-03-02 08:50:05,707][08718] Num frames 100...
+[2023-03-02 08:50:05,851][08718] Num frames 200...
+[2023-03-02 08:50:05,990][08718] Num frames 300...
+[2023-03-02 08:50:06,132][08718] Num frames 400...
+[2023-03-02 08:50:06,269][08718] Num frames 500...
+[2023-03-02 08:50:06,386][08718] Avg episode rewards: #0: 7.440, true rewards: #0: 5.440
+[2023-03-02 08:50:06,386][08718] Avg episode reward: 7.440, avg true_objective: 5.440
+[2023-03-02 08:50:06,471][08718] Num frames 600...
+[2023-03-02 08:50:06,616][08718] Num frames 700...
+[2023-03-02 08:50:06,758][08718] Num frames 800...
+[2023-03-02 08:50:06,895][08718] Num frames 900...
+[2023-03-02 08:50:06,983][08718] Avg episode rewards: #0: 5.640, true rewards: #0: 4.640
+[2023-03-02 08:50:06,984][08718] Avg episode reward: 5.640, avg true_objective: 4.640
+[2023-03-02 08:50:07,086][08718] Num frames 1000...
+[2023-03-02 08:50:07,222][08718] Num frames 1100...
+[2023-03-02 08:50:07,356][08718] Num frames 1200...
+[2023-03-02 08:50:07,500][08718] Num frames 1300...
+[2023-03-02 08:50:07,568][08718] Avg episode rewards: #0: 5.040, true rewards: #0: 4.373
+[2023-03-02 08:50:07,569][08718] Avg episode reward: 5.040, avg true_objective: 4.373
+[2023-03-02 08:50:07,692][08718] Num frames 1400...
+[2023-03-02 08:50:07,828][08718] Num frames 1500...
+[2023-03-02 08:50:07,966][08718] Num frames 1600...
+[2023-03-02 08:50:08,151][08718] Avg episode rewards: #0: 4.740, true rewards: #0: 4.240
+[2023-03-02 08:50:08,151][08718] Avg episode reward: 4.740, avg true_objective: 4.240
+[2023-03-02 08:50:08,156][08718] Num frames 1700...
+[2023-03-02 08:50:08,296][08718] Num frames 1800...
+[2023-03-02 08:50:08,431][08718] Num frames 1900...
+[2023-03-02 08:50:08,575][08718] Num frames 2000...
+[2023-03-02 08:50:08,738][08718] Avg episode rewards: #0: 4.560, true rewards: #0: 4.160
+[2023-03-02 08:50:08,738][08718] Avg episode reward: 4.560, avg true_objective: 4.160
+[2023-03-02 08:50:08,771][08718] Num frames 2100...
+[2023-03-02 08:50:08,912][08718] Num frames 2200...
+[2023-03-02 08:50:09,052][08718] Num frames 2300...
+[2023-03-02 08:50:09,170][08718] Avg episode rewards: #0: 4.238, true rewards: #0: 3.905
+[2023-03-02 08:50:09,170][08718] Avg episode reward: 4.238, avg true_objective: 3.905
+[2023-03-02 08:50:09,250][08718] Num frames 2400...
+[2023-03-02 08:50:09,384][08718] Num frames 2500...
+[2023-03-02 08:50:09,519][08718] Num frames 2600...
+[2023-03-02 08:50:09,654][08718] Num frames 2700...
+[2023-03-02 08:50:09,832][08718] Avg episode rewards: #0: 4.416, true rewards: #0: 3.987
+[2023-03-02 08:50:09,832][08718] Avg episode reward: 4.416, avg true_objective: 3.987
+[2023-03-02 08:50:09,843][08718] Num frames 2800...
+[2023-03-02 08:50:09,975][08718] Num frames 2900...
+[2023-03-02 08:50:10,105][08718] Num frames 3000...
+[2023-03-02 08:50:10,234][08718] Num frames 3100...
+[2023-03-02 08:50:10,390][08718] Avg episode rewards: #0: 4.344, true rewards: #0: 3.969
+[2023-03-02 08:50:10,390][08718] Avg episode reward: 4.344, avg true_objective: 3.969
+[2023-03-02 08:50:10,433][08718] Num frames 3200...
+[2023-03-02 08:50:10,575][08718] Num frames 3300...
+[2023-03-02 08:50:10,709][08718] Num frames 3400...
+[2023-03-02 08:50:10,843][08718] Num frames 3500...
+[2023-03-02 08:50:10,975][08718] Avg episode rewards: #0: 4.288, true rewards: #0: 3.954
+[2023-03-02 08:50:10,976][08718] Avg episode reward: 4.288, avg true_objective: 3.954
+[2023-03-02 08:50:11,036][08718] Num frames 3600...
+[2023-03-02 08:50:11,171][08718] Num frames 3700...
+[2023-03-02 08:50:11,309][08718] Num frames 3800...
+[2023-03-02 08:50:11,448][08718] Num frames 3900...
+[2023-03-02 08:50:11,567][08718] Avg episode rewards: #0: 4.243, true rewards: #0: 3.943
+[2023-03-02 08:50:11,567][08718] Avg episode reward: 4.243, avg true_objective: 3.943
+[2023-03-02 08:50:15,364][08718] Replay video saved to /home/gpu/train_dir/default_experiment/replay.mp4!