kinalmehta
commited on
Commit
•
1933d86
1
Parent(s):
090f58a
pushing model
Browse files- README.md +79 -0
- c51_jax.cleanrl_model +0 -0
- c51_jax.py +304 -0
- events.out.tfevents.1672601060.fedora.38904.0 +3 -0
- poetry.lock +0 -0
- pyproject.toml +178 -0
- replay.mp4 +0 -0
- videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-0.mp4 +0 -0
- videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-1.mp4 +0 -0
- videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-8.mp4 +0 -0
README.md
ADDED
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
tags:
|
3 |
+
- MountainCar-v0
|
4 |
+
- deep-reinforcement-learning
|
5 |
+
- reinforcement-learning
|
6 |
+
- custom-implementation
|
7 |
+
library_name: cleanrl
|
8 |
+
model-index:
|
9 |
+
- name: DQN
|
10 |
+
results:
|
11 |
+
- task:
|
12 |
+
type: reinforcement-learning
|
13 |
+
name: reinforcement-learning
|
14 |
+
dataset:
|
15 |
+
name: MountainCar-v0
|
16 |
+
type: MountainCar-v0
|
17 |
+
metrics:
|
18 |
+
- type: mean_reward
|
19 |
+
value: -200.00 +/- 0.00
|
20 |
+
name: mean_reward
|
21 |
+
verified: false
|
22 |
+
---
|
23 |
+
|
24 |
+
# (CleanRL) **DQN** Agent Playing **MountainCar-v0**
|
25 |
+
|
26 |
+
This is a trained model of a DQN agent playing MountainCar-v0.
|
27 |
+
The model was trained by using [CleanRL](https://github.com/vwxyzjn/cleanrl) and the most up-to-date training code can be
|
28 |
+
found [here](https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/c51_jax.py).
|
29 |
+
|
30 |
+
## Get Started
|
31 |
+
|
32 |
+
To use this model, please install the `cleanrl` package with the following command:
|
33 |
+
|
34 |
+
```
|
35 |
+
pip install "cleanrl[c51_jax]"
|
36 |
+
python -m cleanrl_utils.enjoy --exp-name c51_jax --env-id MountainCar-v0
|
37 |
+
```
|
38 |
+
|
39 |
+
Please refer to the [documentation](https://docs.cleanrl.dev/get-started/zoo/) for more detail.
|
40 |
+
|
41 |
+
|
42 |
+
## Command to reproduce the training
|
43 |
+
|
44 |
+
```bash
|
45 |
+
curl -OL https://huggingface.co/cleanrl/MountainCar-v0-c51_jax-seed1/raw/main/c51_jax.py
|
46 |
+
curl -OL https://huggingface.co/cleanrl/MountainCar-v0-c51_jax-seed1/raw/main/pyproject.toml
|
47 |
+
curl -OL https://huggingface.co/cleanrl/MountainCar-v0-c51_jax-seed1/raw/main/poetry.lock
|
48 |
+
poetry install --all-extras
|
49 |
+
python c51_jax.py --save-model --upload-model --hf-entity cleanrl --env-id MountainCar-v0
|
50 |
+
```
|
51 |
+
|
52 |
+
# Hyperparameters
|
53 |
+
```python
|
54 |
+
{'batch_size': 128,
|
55 |
+
'buffer_size': 10000,
|
56 |
+
'capture_video': False,
|
57 |
+
'end_e': 0.05,
|
58 |
+
'env_id': 'MountainCar-v0',
|
59 |
+
'exp_name': 'c51_jax',
|
60 |
+
'exploration_fraction': 0.5,
|
61 |
+
'gamma': 0.99,
|
62 |
+
'hf_entity': 'cleanrl',
|
63 |
+
'learning_rate': 0.00025,
|
64 |
+
'learning_starts': 10000,
|
65 |
+
'n_atoms': 101,
|
66 |
+
'save_model': True,
|
67 |
+
'seed': 1,
|
68 |
+
'start_e': 1,
|
69 |
+
'target_network_frequency': 500,
|
70 |
+
'total_timesteps': 500000,
|
71 |
+
'track': False,
|
72 |
+
'train_frequency': 10,
|
73 |
+
'upload_model': True,
|
74 |
+
'v_max': 100,
|
75 |
+
'v_min': -100,
|
76 |
+
'wandb_entity': None,
|
77 |
+
'wandb_project_name': 'cleanRL'}
|
78 |
+
```
|
79 |
+
|
c51_jax.cleanrl_model
ADDED
Binary file (146 kB). View file
|
|
c51_jax.py
ADDED
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/c51/#c51_jaxpy
|
2 |
+
import argparse
|
3 |
+
import os
|
4 |
+
import random
|
5 |
+
import time
|
6 |
+
from distutils.util import strtobool
|
7 |
+
|
8 |
+
import flax
|
9 |
+
import flax.linen as nn
|
10 |
+
import gym
|
11 |
+
import jax
|
12 |
+
import jax.numpy as jnp
|
13 |
+
import numpy as np
|
14 |
+
import optax
|
15 |
+
from flax.training.train_state import TrainState
|
16 |
+
from stable_baselines3.common.buffers import ReplayBuffer
|
17 |
+
from torch.utils.tensorboard import SummaryWriter
|
18 |
+
|
19 |
+
|
20 |
+
def parse_args():
|
21 |
+
# fmt: off
|
22 |
+
parser = argparse.ArgumentParser()
|
23 |
+
parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
|
24 |
+
help="the name of this experiment")
|
25 |
+
parser.add_argument("--seed", type=int, default=1,
|
26 |
+
help="seed of the experiment")
|
27 |
+
parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
28 |
+
help="if toggled, this experiment will be tracked with Weights and Biases")
|
29 |
+
parser.add_argument("--wandb-project-name", type=str, default="cleanRL",
|
30 |
+
help="the wandb's project name")
|
31 |
+
parser.add_argument("--wandb-entity", type=str, default=None,
|
32 |
+
help="the entity (team) of wandb's project")
|
33 |
+
parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
34 |
+
help="whether to capture videos of the agent performances (check out `videos` folder)")
|
35 |
+
parser.add_argument("--save-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
36 |
+
help="whether to save model into the `runs/{run_name}` folder")
|
37 |
+
parser.add_argument("--upload-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
38 |
+
help="whether to upload the saved model to huggingface")
|
39 |
+
parser.add_argument("--hf-entity", type=str, default="",
|
40 |
+
help="the user or org name of the model repository from the Hugging Face Hub")
|
41 |
+
|
42 |
+
# Algorithm specific arguments
|
43 |
+
parser.add_argument("--env-id", type=str, default="CartPole-v1",
|
44 |
+
help="the id of the environment")
|
45 |
+
parser.add_argument("--total-timesteps", type=int, default=500000,
|
46 |
+
help="total timesteps of the experiments")
|
47 |
+
parser.add_argument("--learning-rate", type=float, default=2.5e-4,
|
48 |
+
help="the learning rate of the optimizer")
|
49 |
+
parser.add_argument("--n-atoms", type=int, default=101,
|
50 |
+
help="the number of atoms")
|
51 |
+
parser.add_argument("--v-min", type=float, default=-100,
|
52 |
+
help="the number of atoms")
|
53 |
+
parser.add_argument("--v-max", type=float, default=100,
|
54 |
+
help="the number of atoms")
|
55 |
+
parser.add_argument("--buffer-size", type=int, default=10000,
|
56 |
+
help="the replay memory buffer size")
|
57 |
+
parser.add_argument("--gamma", type=float, default=0.99,
|
58 |
+
help="the discount factor gamma")
|
59 |
+
parser.add_argument("--target-network-frequency", type=int, default=500,
|
60 |
+
help="the timesteps it takes to update the target network")
|
61 |
+
parser.add_argument("--batch-size", type=int, default=128,
|
62 |
+
help="the batch size of sample from the reply memory")
|
63 |
+
parser.add_argument("--start-e", type=float, default=1,
|
64 |
+
help="the starting epsilon for exploration")
|
65 |
+
parser.add_argument("--end-e", type=float, default=0.05,
|
66 |
+
help="the ending epsilon for exploration")
|
67 |
+
parser.add_argument("--exploration-fraction", type=float, default=0.5,
|
68 |
+
help="the fraction of `total-timesteps` it takes from start-e to go end-e")
|
69 |
+
parser.add_argument("--learning-starts", type=int, default=10000,
|
70 |
+
help="timestep to start learning")
|
71 |
+
parser.add_argument("--train-frequency", type=int, default=10,
|
72 |
+
help="the frequency of training")
|
73 |
+
args = parser.parse_args()
|
74 |
+
# fmt: on
|
75 |
+
return args
|
76 |
+
|
77 |
+
|
78 |
+
def make_env(env_id, seed, idx, capture_video, run_name):
|
79 |
+
def thunk():
|
80 |
+
env = gym.make(env_id)
|
81 |
+
env = gym.wrappers.RecordEpisodeStatistics(env)
|
82 |
+
if capture_video:
|
83 |
+
if idx == 0:
|
84 |
+
env = gym.wrappers.RecordVideo(env, f"videos/{run_name}")
|
85 |
+
env.seed(seed)
|
86 |
+
env.action_space.seed(seed)
|
87 |
+
env.observation_space.seed(seed)
|
88 |
+
return env
|
89 |
+
|
90 |
+
return thunk
|
91 |
+
|
92 |
+
|
93 |
+
# ALGO LOGIC: initialize agent here:
|
94 |
+
class QNetwork(nn.Module):
|
95 |
+
action_dim: int
|
96 |
+
n_atoms: int
|
97 |
+
|
98 |
+
@nn.compact
|
99 |
+
def __call__(self, x):
|
100 |
+
x = nn.Dense(120)(x)
|
101 |
+
x = nn.relu(x)
|
102 |
+
x = nn.Dense(84)(x)
|
103 |
+
x = nn.relu(x)
|
104 |
+
x = nn.Dense(self.action_dim * self.n_atoms)(x)
|
105 |
+
x = x.reshape((x.shape[0], self.action_dim, self.n_atoms))
|
106 |
+
x = nn.softmax(x, axis=-1) # pmfs
|
107 |
+
return x
|
108 |
+
|
109 |
+
|
110 |
+
class TrainState(TrainState):
|
111 |
+
target_params: flax.core.FrozenDict
|
112 |
+
atoms: jnp.ndarray
|
113 |
+
|
114 |
+
|
115 |
+
def linear_schedule(start_e: float, end_e: float, duration: int, t: int):
|
116 |
+
slope = (end_e - start_e) / duration
|
117 |
+
return max(slope * t + start_e, end_e)
|
118 |
+
|
119 |
+
|
120 |
+
if __name__ == "__main__":
|
121 |
+
args = parse_args()
|
122 |
+
run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
|
123 |
+
if args.track:
|
124 |
+
import wandb
|
125 |
+
|
126 |
+
wandb.init(
|
127 |
+
project=args.wandb_project_name,
|
128 |
+
entity=args.wandb_entity,
|
129 |
+
sync_tensorboard=True,
|
130 |
+
config=vars(args),
|
131 |
+
name=run_name,
|
132 |
+
monitor_gym=True,
|
133 |
+
save_code=True,
|
134 |
+
)
|
135 |
+
writer = SummaryWriter(f"runs/{run_name}")
|
136 |
+
writer.add_text(
|
137 |
+
"hyperparameters",
|
138 |
+
"|param|value|\n|-|-|\n%s" % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
|
139 |
+
)
|
140 |
+
|
141 |
+
# TRY NOT TO MODIFY: seeding
|
142 |
+
random.seed(args.seed)
|
143 |
+
np.random.seed(args.seed)
|
144 |
+
key = jax.random.PRNGKey(args.seed)
|
145 |
+
key, q_key = jax.random.split(key, 2)
|
146 |
+
|
147 |
+
# env setup
|
148 |
+
envs = gym.vector.SyncVectorEnv([make_env(args.env_id, args.seed, 0, args.capture_video, run_name)])
|
149 |
+
assert isinstance(envs.single_action_space, gym.spaces.Discrete), "only discrete action space is supported"
|
150 |
+
|
151 |
+
obs = envs.reset()
|
152 |
+
q_network = QNetwork(action_dim=envs.single_action_space.n, n_atoms=args.n_atoms)
|
153 |
+
q_state = TrainState.create(
|
154 |
+
apply_fn=q_network.apply,
|
155 |
+
params=q_network.init(q_key, obs),
|
156 |
+
target_params=q_network.init(q_key, obs),
|
157 |
+
# directly using jnp.linspace leads to numerical errors
|
158 |
+
atoms=jnp.asarray(np.linspace(args.v_min, args.v_max, num=args.n_atoms)),
|
159 |
+
tx=optax.adam(learning_rate=args.learning_rate, eps=0.01 / args.batch_size),
|
160 |
+
)
|
161 |
+
q_network.apply = jax.jit(q_network.apply)
|
162 |
+
# This step is not necessary as init called on same observation and key will always lead to same initializations
|
163 |
+
q_state = q_state.replace(target_params=optax.incremental_update(q_state.params, q_state.target_params, 1))
|
164 |
+
|
165 |
+
rb = ReplayBuffer(
|
166 |
+
args.buffer_size,
|
167 |
+
envs.single_observation_space,
|
168 |
+
envs.single_action_space,
|
169 |
+
"cpu",
|
170 |
+
handle_timeout_termination=True,
|
171 |
+
)
|
172 |
+
|
173 |
+
@jax.jit
|
174 |
+
def update(q_state, observations, actions, next_observations, rewards, dones):
|
175 |
+
next_pmfs = q_network.apply(q_state.target_params, next_observations) # (batch_size, num_actions, num_atoms)
|
176 |
+
next_vals = (next_pmfs * q_state.atoms).sum(axis=-1) # (batch_size, num_actions)
|
177 |
+
next_action = jnp.argmax(next_vals, axis=-1) # (batch_size,)
|
178 |
+
next_pmfs = next_pmfs[np.arange(next_pmfs.shape[0]), next_action]
|
179 |
+
next_atoms = rewards + args.gamma * q_state.atoms * (1 - dones)
|
180 |
+
# projection
|
181 |
+
delta_z = q_state.atoms[1] - q_state.atoms[0]
|
182 |
+
tz = jnp.clip(next_atoms, a_min=(args.v_min), a_max=(args.v_max))
|
183 |
+
|
184 |
+
b = (tz - args.v_min) / delta_z
|
185 |
+
l = jnp.clip(jnp.floor(b), a_min=0, a_max=args.n_atoms - 1)
|
186 |
+
u = jnp.clip(jnp.ceil(b), a_min=0, a_max=args.n_atoms - 1)
|
187 |
+
# (l == u).astype(jnp.float) handles the case where bj is exactly an integer
|
188 |
+
# example bj = 1, then the upper ceiling should be uj= 2, and lj= 1
|
189 |
+
d_m_l = (u + (l == u).astype(jnp.float32) - b) * next_pmfs
|
190 |
+
d_m_u = (b - l) * next_pmfs
|
191 |
+
target_pmfs = jnp.zeros_like(next_pmfs)
|
192 |
+
|
193 |
+
def project_to_bins(i, val):
|
194 |
+
val = val.at[i, l[i].astype(jnp.int32)].add(d_m_l[i])
|
195 |
+
val = val.at[i, u[i].astype(jnp.int32)].add(d_m_u[i])
|
196 |
+
return val
|
197 |
+
|
198 |
+
target_pmfs = jax.lax.fori_loop(0, target_pmfs.shape[0], project_to_bins, target_pmfs)
|
199 |
+
|
200 |
+
def loss(q_params, observations, actions, target_pmfs):
|
201 |
+
pmfs = q_network.apply(q_params, observations)
|
202 |
+
old_pmfs = pmfs[np.arange(pmfs.shape[0]), actions.squeeze()]
|
203 |
+
|
204 |
+
old_pmfs_l = jnp.clip(old_pmfs, a_min=1e-5, a_max=1 - 1e-5)
|
205 |
+
loss = (-(target_pmfs * jnp.log(old_pmfs_l)).sum(-1)).mean()
|
206 |
+
return loss, (old_pmfs * q_state.atoms).sum(-1)
|
207 |
+
|
208 |
+
(loss_value, old_values), grads = jax.value_and_grad(loss, has_aux=True)(
|
209 |
+
q_state.params, observations, actions, target_pmfs
|
210 |
+
)
|
211 |
+
q_state = q_state.apply_gradients(grads=grads)
|
212 |
+
return loss_value, old_values, q_state
|
213 |
+
|
214 |
+
start_time = time.time()
|
215 |
+
|
216 |
+
# TRY NOT TO MODIFY: start the game
|
217 |
+
obs = envs.reset()
|
218 |
+
for global_step in range(args.total_timesteps):
|
219 |
+
# ALGO LOGIC: put action logic here
|
220 |
+
epsilon = linear_schedule(args.start_e, args.end_e, args.exploration_fraction * args.total_timesteps, global_step)
|
221 |
+
if random.random() < epsilon:
|
222 |
+
actions = np.array([envs.single_action_space.sample() for _ in range(envs.num_envs)])
|
223 |
+
else:
|
224 |
+
pmfs = q_network.apply(q_state.params, obs)
|
225 |
+
q_vals = (pmfs * q_state.atoms).sum(axis=-1)
|
226 |
+
actions = q_vals.argmax(axis=-1)
|
227 |
+
actions = jax.device_get(actions)
|
228 |
+
|
229 |
+
# TRY NOT TO MODIFY: execute the game and log data.
|
230 |
+
next_obs, rewards, dones, infos = envs.step(actions)
|
231 |
+
|
232 |
+
# TRY NOT TO MODIFY: record rewards for plotting purposes
|
233 |
+
for info in infos:
|
234 |
+
if "episode" in info.keys():
|
235 |
+
print(f"global_step={global_step}, episodic_return={info['episode']['r']}")
|
236 |
+
writer.add_scalar("charts/episodic_return", info["episode"]["r"], global_step)
|
237 |
+
writer.add_scalar("charts/episodic_length", info["episode"]["l"], global_step)
|
238 |
+
writer.add_scalar("charts/epsilon", epsilon, global_step)
|
239 |
+
break
|
240 |
+
|
241 |
+
# TRY NOT TO MODIFY: save data to reply buffer; handle `terminal_observation`
|
242 |
+
real_next_obs = next_obs.copy()
|
243 |
+
for idx, d in enumerate(dones):
|
244 |
+
if d:
|
245 |
+
real_next_obs[idx] = infos[idx]["terminal_observation"]
|
246 |
+
rb.add(obs, real_next_obs, actions, rewards, dones, infos)
|
247 |
+
|
248 |
+
# TRY NOT TO MODIFY: CRUCIAL step easy to overlook
|
249 |
+
obs = next_obs
|
250 |
+
|
251 |
+
# ALGO LOGIC: training.
|
252 |
+
if global_step > args.learning_starts and global_step % args.train_frequency == 0:
|
253 |
+
data = rb.sample(args.batch_size)
|
254 |
+
loss, old_val, q_state = update(
|
255 |
+
q_state,
|
256 |
+
data.observations.numpy(),
|
257 |
+
data.actions.numpy(),
|
258 |
+
data.next_observations.numpy(),
|
259 |
+
data.rewards.numpy(),
|
260 |
+
data.dones.numpy(),
|
261 |
+
)
|
262 |
+
|
263 |
+
if global_step % 100 == 0:
|
264 |
+
writer.add_scalar("losses/loss", jax.device_get(loss), global_step)
|
265 |
+
writer.add_scalar("losses/q_values", jax.device_get(old_val.mean()), global_step)
|
266 |
+
print("SPS:", int(global_step / (time.time() - start_time)))
|
267 |
+
writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
|
268 |
+
|
269 |
+
# update the target network
|
270 |
+
if global_step % args.target_network_frequency == 0:
|
271 |
+
q_state = q_state.replace(target_params=optax.incremental_update(q_state.params, q_state.target_params, 1))
|
272 |
+
|
273 |
+
if args.save_model:
|
274 |
+
model_path = f"runs/{run_name}/{args.exp_name}.cleanrl_model"
|
275 |
+
model_data = {
|
276 |
+
"model_weights": q_state.params,
|
277 |
+
"args": vars(args),
|
278 |
+
}
|
279 |
+
with open(model_path, "wb") as f:
|
280 |
+
f.write(flax.serialization.to_bytes(model_data))
|
281 |
+
print(f"model saved to {model_path}")
|
282 |
+
from cleanrl_utils.evals.c51_jax_eval import evaluate
|
283 |
+
|
284 |
+
episodic_returns = evaluate(
|
285 |
+
model_path,
|
286 |
+
make_env,
|
287 |
+
args.env_id,
|
288 |
+
eval_episodes=10,
|
289 |
+
run_name=f"{run_name}-eval",
|
290 |
+
Model=QNetwork,
|
291 |
+
epsilon=0.05,
|
292 |
+
)
|
293 |
+
for idx, episodic_return in enumerate(episodic_returns):
|
294 |
+
writer.add_scalar("eval/episodic_return", episodic_return, idx)
|
295 |
+
|
296 |
+
if args.upload_model:
|
297 |
+
from cleanrl_utils.huggingface import push_to_hub
|
298 |
+
|
299 |
+
repo_name = f"{args.env_id}-{args.exp_name}-seed{args.seed}"
|
300 |
+
repo_id = f"{args.hf_entity}/{repo_name}" if args.hf_entity else repo_name
|
301 |
+
push_to_hub(args, episodic_returns, repo_id, "DQN", f"runs/{run_name}", f"videos/{run_name}-eval")
|
302 |
+
|
303 |
+
envs.close()
|
304 |
+
writer.close()
|
events.out.tfevents.1672601060.fedora.38904.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:41c447dde582a32335d329b92e4593751ae8b6921e1ec986ca69f65fc5f6ade5
|
3 |
+
size 1210173
|
poetry.lock
ADDED
The diff for this file is too large to render.
See raw diff
|
|
pyproject.toml
ADDED
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[tool.poetry]
|
2 |
+
name = "cleanrl-test"
|
3 |
+
version = "1.1.0"
|
4 |
+
description = "High-quality single file implementation of Deep Reinforcement Learning algorithms with research-friendly features"
|
5 |
+
authors = ["Costa Huang <[email protected]>"]
|
6 |
+
packages = [
|
7 |
+
{ include = "cleanrl" },
|
8 |
+
{ include = "cleanrl_utils" },
|
9 |
+
]
|
10 |
+
keywords = ["reinforcement", "machine", "learning", "research"]
|
11 |
+
license="MIT"
|
12 |
+
readme = "README.md"
|
13 |
+
|
14 |
+
[tool.poetry.dependencies]
|
15 |
+
python = ">=3.7.1,<3.10"
|
16 |
+
tensorboard = "^2.10.0"
|
17 |
+
wandb = "^0.13.6"
|
18 |
+
gym = "0.23.1"
|
19 |
+
torch = ">=1.12.1"
|
20 |
+
stable-baselines3 = "1.2.0"
|
21 |
+
gymnasium = "^0.26.3"
|
22 |
+
moviepy = "^1.0.3"
|
23 |
+
pygame = "2.1.0"
|
24 |
+
huggingface-hub = "^0.11.1"
|
25 |
+
|
26 |
+
ale-py = {version = "0.7.4", optional = true}
|
27 |
+
AutoROM = {extras = ["accept-rom-license"], version = "^0.4.2"}
|
28 |
+
opencv-python = {version = "^4.6.0.66", optional = true}
|
29 |
+
pybullet = {version = "3.1.8", optional = true}
|
30 |
+
procgen = {version = "^0.10.7", optional = true}
|
31 |
+
pytest = {version = "^7.1.3", optional = true}
|
32 |
+
mujoco = {version = "^2.2", optional = true}
|
33 |
+
imageio = {version = "^2.14.1", optional = true}
|
34 |
+
free-mujoco-py = {version = "^2.1.6", optional = true}
|
35 |
+
mkdocs-material = {version = "^8.4.3", optional = true}
|
36 |
+
markdown-include = {version = "^0.7.0", optional = true}
|
37 |
+
jax = {version = "^0.3.17", optional = true}
|
38 |
+
jaxlib = {version = "^0.3.15", optional = true}
|
39 |
+
flax = {version = "^0.6.0", optional = true}
|
40 |
+
optuna = {version = "^3.0.1", optional = true}
|
41 |
+
optuna-dashboard = {version = "^0.7.2", optional = true}
|
42 |
+
rich = {version = "<12.0", optional = true}
|
43 |
+
envpool = {version = "^0.6.4", optional = true}
|
44 |
+
PettingZoo = {version = "1.18.1", optional = true}
|
45 |
+
SuperSuit = {version = "3.4.0", optional = true}
|
46 |
+
multi-agent-ale-py = {version = "0.1.11", optional = true}
|
47 |
+
boto3 = {version = "^1.24.70", optional = true}
|
48 |
+
awscli = {version = "^1.25.71", optional = true}
|
49 |
+
shimmy = {version = "^0.1.0", optional = true}
|
50 |
+
dm-control = {version = "^1.0.8", optional = true}
|
51 |
+
|
52 |
+
[tool.poetry.group.dev.dependencies]
|
53 |
+
pre-commit = "^2.20.0"
|
54 |
+
|
55 |
+
[tool.poetry.group.atari]
|
56 |
+
optional = true
|
57 |
+
[tool.poetry.group.atari.dependencies]
|
58 |
+
ale-py = "0.7.4"
|
59 |
+
AutoROM = {extras = ["accept-rom-license"], version = "^0.4.2"}
|
60 |
+
opencv-python = "^4.6.0.66"
|
61 |
+
|
62 |
+
[tool.poetry.group.pybullet]
|
63 |
+
optional = true
|
64 |
+
[tool.poetry.group.pybullet.dependencies]
|
65 |
+
pybullet = "3.1.8"
|
66 |
+
|
67 |
+
[tool.poetry.group.procgen]
|
68 |
+
optional = true
|
69 |
+
[tool.poetry.group.procgen.dependencies]
|
70 |
+
procgen = "^0.10.7"
|
71 |
+
|
72 |
+
[tool.poetry.group.pytest]
|
73 |
+
optional = true
|
74 |
+
[tool.poetry.group.pytest.dependencies]
|
75 |
+
pytest = "^7.1.3"
|
76 |
+
|
77 |
+
[tool.poetry.group.mujoco]
|
78 |
+
optional = true
|
79 |
+
[tool.poetry.group.mujoco.dependencies]
|
80 |
+
mujoco = "^2.2"
|
81 |
+
imageio = "^2.14.1"
|
82 |
+
|
83 |
+
[tool.poetry.group.mujoco_py]
|
84 |
+
optional = true
|
85 |
+
[tool.poetry.group.mujoco_py.dependencies]
|
86 |
+
free-mujoco-py = "^2.1.6"
|
87 |
+
|
88 |
+
[tool.poetry.group.docs]
|
89 |
+
optional = true
|
90 |
+
[tool.poetry.group.docs.dependencies]
|
91 |
+
mkdocs-material = "^8.4.3"
|
92 |
+
markdown-include = "^0.7.0"
|
93 |
+
|
94 |
+
[tool.poetry.group.jax]
|
95 |
+
optional = true
|
96 |
+
[tool.poetry.group.jax.dependencies]
|
97 |
+
jax = "^0.3.17"
|
98 |
+
jaxlib = "^0.3.15"
|
99 |
+
flax = "^0.6.0"
|
100 |
+
|
101 |
+
[tool.poetry.group.optuna]
|
102 |
+
optional = true
|
103 |
+
[tool.poetry.group.optuna.dependencies]
|
104 |
+
optuna = "^3.0.1"
|
105 |
+
optuna-dashboard = "^0.7.2"
|
106 |
+
rich = "<12.0"
|
107 |
+
|
108 |
+
[tool.poetry.group.envpool]
|
109 |
+
optional = true
|
110 |
+
[tool.poetry.group.envpool.dependencies]
|
111 |
+
envpool = "^0.6.4"
|
112 |
+
|
113 |
+
[tool.poetry.group.pettingzoo]
|
114 |
+
optional = true
|
115 |
+
[tool.poetry.group.pettingzoo.dependencies]
|
116 |
+
PettingZoo = "1.18.1"
|
117 |
+
SuperSuit = "3.4.0"
|
118 |
+
multi-agent-ale-py = "0.1.11"
|
119 |
+
|
120 |
+
[tool.poetry.group.cloud]
|
121 |
+
optional = true
|
122 |
+
[tool.poetry.group.cloud.dependencies]
|
123 |
+
boto3 = "^1.24.70"
|
124 |
+
awscli = "^1.25.71"
|
125 |
+
|
126 |
+
[tool.poetry.group.isaacgym]
|
127 |
+
optional = true
|
128 |
+
[tool.poetry.group.isaacgym.dependencies]
|
129 |
+
isaacgymenvs = {git = "https://github.com/vwxyzjn/IsaacGymEnvs.git", rev = "poetry"}
|
130 |
+
isaacgym = {path = "cleanrl/ppo_continuous_action_isaacgym/isaacgym", develop = true}
|
131 |
+
|
132 |
+
[tool.poetry.group.dm_control]
|
133 |
+
optional = true
|
134 |
+
[tool.poetry.group.dm_control.dependencies]
|
135 |
+
shimmy = "^0.1.0"
|
136 |
+
dm-control = "^1.0.8"
|
137 |
+
mujoco = "^2.2"
|
138 |
+
|
139 |
+
[build-system]
|
140 |
+
requires = ["poetry-core"]
|
141 |
+
build-backend = "poetry.core.masonry.api"
|
142 |
+
|
143 |
+
[tool.poetry.extras]
|
144 |
+
atari = ["ale-py", "AutoROM", "opencv-python"]
|
145 |
+
pybullet = ["pybullet"]
|
146 |
+
procgen = ["procgen"]
|
147 |
+
plot = ["pandas", "seaborn"]
|
148 |
+
pytest = ["pytest"]
|
149 |
+
mujoco = ["mujoco", "imageio"]
|
150 |
+
mujoco_py = ["free-mujoco-py"]
|
151 |
+
jax = ["jax", "jaxlib", "flax"]
|
152 |
+
docs = ["mkdocs-material", "markdown-include"]
|
153 |
+
envpool = ["envpool"]
|
154 |
+
optuna = ["optuna", "optuna-dashboard", "rich"]
|
155 |
+
pettingzoo = ["PettingZoo", "SuperSuit", "multi-agent-ale-py"]
|
156 |
+
cloud = ["boto3", "awscli"]
|
157 |
+
dm_control = ["shimmy", "dm-control", "mujoco"]
|
158 |
+
|
159 |
+
# dependencies for algorithm variant (useful when you want to run a specific algorithm)
|
160 |
+
dqn = []
|
161 |
+
dqn_atari = ["ale-py", "AutoROM", "opencv-python"]
|
162 |
+
dqn_jax = ["jax", "jaxlib", "flax"]
|
163 |
+
dqn_atari_jax = [
|
164 |
+
"ale-py", "AutoROM", "opencv-python", # atari
|
165 |
+
"jax", "jaxlib", "flax" # jax
|
166 |
+
]
|
167 |
+
c51 = []
|
168 |
+
c51_atari = ["ale-py", "AutoROM", "opencv-python"]
|
169 |
+
c51_jax = ["jax", "jaxlib", "flax"]
|
170 |
+
c51_atari_jax = [
|
171 |
+
"ale-py", "AutoROM", "opencv-python", # atari
|
172 |
+
"jax", "jaxlib", "flax" # jax
|
173 |
+
]
|
174 |
+
ppo_atari_envpool_xla_jax_scan = [
|
175 |
+
"ale-py", "AutoROM", "opencv-python", # atari
|
176 |
+
"jax", "jaxlib", "flax", # jax
|
177 |
+
"envpool", # envpool
|
178 |
+
]
|
replay.mp4
ADDED
Binary file (43.4 kB). View file
|
|
videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-0.mp4
ADDED
Binary file (42.6 kB). View file
|
|
videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-1.mp4
ADDED
Binary file (35.4 kB). View file
|
|
videos/MountainCar-v0__c51_jax__1__1672601060-eval/rl-video-episode-8.mp4
ADDED
Binary file (43.4 kB). View file
|
|