omnirl-framework 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omnirl_framework-0.3.0/omnirl_framework.egg-info → omnirl_framework-0.3.2}/PKG-INFO +2 -2
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/README.md +1 -1
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/policy.py +6 -8
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/policy.py +6 -9
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/policy.py +8 -12
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/policy.py +8 -12
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/policy.py +5 -7
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/policy.py +8 -12
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/policy.py +6 -9
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/policy.py +6 -9
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/prioritized_replay_buffer.py +53 -34
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/replay_buffer.py +51 -31
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/rollout_buffer.py +32 -45
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/collection.py +18 -13
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/sampler.py +3 -4
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2/omnirl_framework.egg-info}/PKG-INFO +2 -2
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/pyproject.toml +1 -1
- omnirl_framework-0.3.2/tests/test_gymnasium.py +127 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_integration.py +19 -2
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_runtime.py +55 -39
- omnirl_framework-0.3.0/tests/test_gymnasium.py +0 -45
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/LICENSE +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/learner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/adapter.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/builder.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/evaluation.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/spec.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/actors.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/cnn.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/mlp.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/resnet.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/critics.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/evaluator.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/file_writer.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/logger.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/runner.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/weight_store.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/utils/__init__.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/utils/seed.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/SOURCES.txt +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/dependency_links.txt +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/requires.txt +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/top_level.txt +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/requirements/base.txt +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/setup.cfg +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_networks.py +0 -0
- {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_structure.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: omnirl-framework
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: A distributed reinforcement learning framework.
|
|
5
5
|
Author-email: Huang Chenbiao <a1057951125@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -197,7 +197,7 @@ __all__ = ["Learner", "Memory", "Policy"]
|
|
|
197
197
|
The implementation should provide:
|
|
198
198
|
|
|
199
199
|
- `Learner`: a Ray actor that implements learning, checkpointing, and versioned weight access. Existing learners can be used as interface examples.
|
|
200
|
-
- `Policy`: a local inference class with deterministic `choose_action` and exploratory `sample_action`
|
|
200
|
+
- `Policy`: a local inference class with deterministic `choose_action(observation)` for a single observation and exploratory `sample_action(observations)` for a batch. Sampling inputs and outputs must retain the leading batch dimension, including when the batch size is 1.
|
|
201
201
|
- `Memory`: one of the buffers in `omnirl.buffers`, or a compatible new Ray actor.
|
|
202
202
|
- Network definitions under `omnirl.networks` when the algorithm needs a new architecture.
|
|
203
203
|
- An agent YAML whose `agent.module` points to `omnirl.algorithms.<name>` and whose `modules` section supplies the constructor parameters.
|
|
@@ -169,7 +169,7 @@ __all__ = ["Learner", "Memory", "Policy"]
|
|
|
169
169
|
The implementation should provide:
|
|
170
170
|
|
|
171
171
|
- `Learner`: a Ray actor that implements learning, checkpointing, and versioned weight access. Existing learners can be used as interface examples.
|
|
172
|
-
- `Policy`: a local inference class with deterministic `choose_action` and exploratory `sample_action`
|
|
172
|
+
- `Policy`: a local inference class with deterministic `choose_action(observation)` for a single observation and exploratory `sample_action(observations)` for a batch. Sampling inputs and outputs must retain the leading batch dimension, including when the batch size is 1.
|
|
173
173
|
- `Memory`: one of the buffers in `omnirl.buffers`, or a compatible new Ray actor.
|
|
174
174
|
- Network definitions under `omnirl.networks` when the algorithm needs a new architecture.
|
|
175
175
|
- An agent YAML whose `agent.module` points to `omnirl.algorithms.<name>` and whose `modules` section supplies the constructor parameters.
|
|
@@ -81,22 +81,20 @@ class Policy:
|
|
|
81
81
|
return int(action.item())
|
|
82
82
|
|
|
83
83
|
@torch.no_grad()
|
|
84
|
-
def sample_action(self,
|
|
85
|
-
"""
|
|
86
|
-
|
|
87
|
-
state = torch.as_tensor(
|
|
88
|
-
state = state if batched else state.unsqueeze(0)
|
|
84
|
+
def sample_action(self, observations):
|
|
85
|
+
"""Sample actions for observations shaped [batch, *state_shape]."""
|
|
86
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
87
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
89
88
|
|
|
90
89
|
if self.is_continuous:
|
|
91
90
|
action_mean, action_log_std = self.actor(state)
|
|
92
91
|
action = Normal(action_mean, action_log_std.exp()).sample()
|
|
93
|
-
action = action if batched else action[0]
|
|
94
92
|
env_action = torch.tanh(action) * self.action_scale + self.action_bias
|
|
95
93
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
96
94
|
else:
|
|
97
95
|
action_logits = self.actor(state)
|
|
98
96
|
action = Categorical(logits=action_logits).sample()
|
|
99
|
-
action = action.cpu().numpy()
|
|
97
|
+
action = action.cpu().numpy()
|
|
100
98
|
env_action = action
|
|
101
99
|
|
|
102
|
-
return {"
|
|
100
|
+
return {"action": action, "env_action": env_action, "extra_data": {}}
|
|
@@ -84,11 +84,10 @@ class Policy:
|
|
|
84
84
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
85
85
|
|
|
86
86
|
@torch.no_grad()
|
|
87
|
-
def sample_action(self,
|
|
88
|
-
"""Select epsilon-greedy actions
|
|
89
|
-
|
|
90
|
-
state = torch.as_tensor(
|
|
91
|
-
state = state if batched else state.unsqueeze(0)
|
|
87
|
+
def sample_action(self, observations):
|
|
88
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
89
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
90
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
92
91
|
|
|
93
92
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
94
93
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -100,5 +99,4 @@ class Policy:
|
|
|
100
99
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
101
100
|
action = np.where(random_mask, action, greedy_action)
|
|
102
101
|
|
|
103
|
-
action
|
|
104
|
-
return {"env_action": action, "data": {"action": action}}
|
|
102
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -74,21 +74,18 @@ class Policy:
|
|
|
74
74
|
return action.squeeze(0).cpu().numpy()
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Sample noisy normalized actions for a batch of observations."""
|
|
79
79
|
if self.random_warmup and self.version == 0:
|
|
80
|
-
scale = self.action_scale
|
|
81
|
-
scale = scale.expand(len(observation), -1) if batched else scale
|
|
80
|
+
scale = self.action_scale.expand(len(observations), -1)
|
|
82
81
|
action = torch.empty_like(scale).uniform_(-1, 1)
|
|
83
82
|
else:
|
|
84
|
-
|
|
85
|
-
state = torch.as_tensor(
|
|
86
|
-
state = state if batched else state.unsqueeze(0)
|
|
83
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
84
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
87
85
|
action = self.actor(state)
|
|
88
86
|
noise = torch.randn_like(action) * self.action_noise
|
|
89
87
|
action = torch.clamp(action + noise, -1, 1)
|
|
90
|
-
action = action if batched else action[0]
|
|
91
88
|
|
|
92
89
|
env_action = action * self.action_scale + self.action_bias
|
|
93
90
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
94
|
-
return {"
|
|
91
|
+
return {"action": action, "env_action": env_action, "extra_data": {}}
|
|
@@ -74,11 +74,10 @@ class Policy:
|
|
|
74
74
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""Select epsilon-greedy actions
|
|
79
|
-
|
|
80
|
-
state = torch.as_tensor(
|
|
81
|
-
state = state if batched else state.unsqueeze(0)
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
79
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
80
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
82
81
|
|
|
83
82
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
84
83
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -88,5 +87,4 @@ class Policy:
|
|
|
88
87
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
89
88
|
action = np.where(random_mask, action, greedy_action)
|
|
90
89
|
|
|
91
|
-
action
|
|
92
|
-
return {"env_action": action, "data": {"action": action}}
|
|
90
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -74,11 +74,10 @@ class Policy:
|
|
|
74
74
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""Select epsilon-greedy actions
|
|
79
|
-
|
|
80
|
-
state = torch.as_tensor(
|
|
81
|
-
state = state if batched else state.unsqueeze(0)
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
79
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
80
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
82
81
|
|
|
83
82
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
84
83
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -88,5 +87,4 @@ class Policy:
|
|
|
88
87
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
89
88
|
action = np.where(random_mask, action, greedy_action)
|
|
90
89
|
|
|
91
|
-
action
|
|
92
|
-
return {"env_action": action, "data": {"action": action}}
|
|
90
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -74,11 +74,10 @@ class Policy:
|
|
|
74
74
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""Select epsilon-greedy actions
|
|
79
|
-
|
|
80
|
-
state = torch.as_tensor(
|
|
81
|
-
state = state if batched else state.unsqueeze(0)
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
79
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
80
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
82
81
|
|
|
83
82
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
84
83
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -88,5 +87,4 @@ class Policy:
|
|
|
88
87
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
89
88
|
action = np.where(random_mask, action, greedy_action)
|
|
90
89
|
|
|
91
|
-
action
|
|
92
|
-
return {"env_action": action, "data": {"action": action}}
|
|
90
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -81,18 +81,16 @@ class Policy:
|
|
|
81
81
|
return int(action.item())
|
|
82
82
|
|
|
83
83
|
@torch.no_grad()
|
|
84
|
-
def sample_action(self,
|
|
85
|
-
"""
|
|
86
|
-
|
|
87
|
-
state = torch.as_tensor(
|
|
88
|
-
state = state if batched else state.unsqueeze(0)
|
|
84
|
+
def sample_action(self, observations):
|
|
85
|
+
"""Sample actions and log probabilities for a batch of observations."""
|
|
86
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
87
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
89
88
|
|
|
90
89
|
if self.is_continuous:
|
|
91
90
|
action_mean, action_log_std = self.actor(state)
|
|
92
91
|
dist = Normal(action_mean, action_log_std.exp())
|
|
93
92
|
action = dist.sample()
|
|
94
93
|
log_prob = dist.log_prob(action).sum(dim=-1)
|
|
95
|
-
action = action if batched else action[0]
|
|
96
94
|
env_action = torch.tanh(action) * self.action_scale + self.action_bias
|
|
97
95
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
98
96
|
else:
|
|
@@ -100,11 +98,9 @@ class Policy:
|
|
|
100
98
|
dist = Categorical(logits=action_logits)
|
|
101
99
|
action = dist.sample()
|
|
102
100
|
log_prob = dist.log_prob(action)
|
|
103
|
-
action = action.cpu().numpy()
|
|
101
|
+
action = action.cpu().numpy()
|
|
104
102
|
env_action = action
|
|
105
103
|
|
|
106
|
-
log_prob = log_prob.cpu().numpy()
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
"data": {"action": action, "log_prob": log_prob},
|
|
110
|
-
}
|
|
104
|
+
log_prob = log_prob.cpu().numpy()
|
|
105
|
+
extra_data = {"log_probs": log_prob}
|
|
106
|
+
return {"action": action, "env_action": env_action, "extra_data": extra_data}
|
|
@@ -74,11 +74,10 @@ class Policy:
|
|
|
74
74
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""Select epsilon-greedy actions
|
|
79
|
-
|
|
80
|
-
state = torch.as_tensor(
|
|
81
|
-
state = state if batched else state.unsqueeze(0)
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
79
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
80
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
82
81
|
|
|
83
82
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
84
83
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -88,5 +87,4 @@ class Policy:
|
|
|
88
87
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
89
88
|
action = np.where(random_mask, action, greedy_action)
|
|
90
89
|
|
|
91
|
-
action
|
|
92
|
-
return {"env_action": action, "data": {"action": action}}
|
|
90
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -81,18 +81,16 @@ class Policy:
|
|
|
81
81
|
return int(action.item())
|
|
82
82
|
|
|
83
83
|
@torch.no_grad()
|
|
84
|
-
def sample_action(self,
|
|
85
|
-
"""
|
|
86
|
-
|
|
87
|
-
state = torch.as_tensor(
|
|
88
|
-
state = state if batched else state.unsqueeze(0)
|
|
84
|
+
def sample_action(self, observations):
|
|
85
|
+
"""Sample actions and log probabilities for a batch of observations."""
|
|
86
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
87
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
89
88
|
|
|
90
89
|
if self.is_continuous:
|
|
91
90
|
action_mean, action_log_std = self.actor(state)
|
|
92
91
|
dist = Normal(action_mean, action_log_std.exp())
|
|
93
92
|
action = dist.sample()
|
|
94
93
|
log_prob = dist.log_prob(action).sum(dim=-1)
|
|
95
|
-
action = action if batched else action[0]
|
|
96
94
|
env_action = torch.tanh(action) * self.action_scale + self.action_bias
|
|
97
95
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
98
96
|
else:
|
|
@@ -100,11 +98,9 @@ class Policy:
|
|
|
100
98
|
dist = Categorical(logits=action_logits)
|
|
101
99
|
action = dist.sample()
|
|
102
100
|
log_prob = dist.log_prob(action)
|
|
103
|
-
action = action.cpu().numpy()
|
|
101
|
+
action = action.cpu().numpy()
|
|
104
102
|
env_action = action
|
|
105
103
|
|
|
106
|
-
log_prob = log_prob.cpu().numpy()
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
"data": {"action": action, "log_prob": log_prob},
|
|
110
|
-
}
|
|
104
|
+
log_prob = log_prob.cpu().numpy()
|
|
105
|
+
extra_data = {"log_probs": log_prob}
|
|
106
|
+
return {"action": action, "env_action": env_action, "extra_data": extra_data}
|
|
@@ -78,11 +78,10 @@ class Policy:
|
|
|
78
78
|
return int(torch.argmax(q_values, dim=-1).item())
|
|
79
79
|
|
|
80
80
|
@torch.no_grad()
|
|
81
|
-
def sample_action(self,
|
|
82
|
-
"""Select epsilon-greedy actions
|
|
83
|
-
|
|
84
|
-
state = torch.as_tensor(
|
|
85
|
-
state = state if batched else state.unsqueeze(0)
|
|
81
|
+
def sample_action(self, observations):
|
|
82
|
+
"""Select epsilon-greedy actions for a batch of observations."""
|
|
83
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
84
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
86
85
|
|
|
87
86
|
action = torch.randint(self.action_dim, (len(state),)).numpy()
|
|
88
87
|
if not (self.random_warmup and self.version == 0):
|
|
@@ -92,5 +91,4 @@ class Policy:
|
|
|
92
91
|
greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
|
|
93
92
|
action = np.where(random_mask, action, greedy_action)
|
|
94
93
|
|
|
95
|
-
action
|
|
96
|
-
return {"env_action": action, "data": {"action": action}}
|
|
94
|
+
return {"action": action, "env_action": action, "extra_data": {}}
|
|
@@ -85,33 +85,29 @@ class Policy:
|
|
|
85
85
|
return int(action.item())
|
|
86
86
|
|
|
87
87
|
@torch.no_grad()
|
|
88
|
-
def sample_action(self,
|
|
89
|
-
"""
|
|
88
|
+
def sample_action(self, observations):
|
|
89
|
+
"""Sample stochastic actions for a batch of observations."""
|
|
90
90
|
if self.random_warmup and self.version == 0:
|
|
91
91
|
if self.is_continuous:
|
|
92
|
-
scale = self.action_scale
|
|
93
|
-
scale = scale.expand(len(observation), -1) if batched else scale
|
|
92
|
+
scale = self.action_scale.expand(len(observations), -1)
|
|
94
93
|
action = torch.empty_like(scale).uniform_(-1, 1)
|
|
95
94
|
else:
|
|
96
|
-
|
|
97
|
-
action = torch.randint(self.action_dim, (size,))
|
|
95
|
+
action = torch.randint(self.action_dim, (len(observations),))
|
|
98
96
|
else:
|
|
99
|
-
|
|
100
|
-
state = torch.as_tensor(
|
|
101
|
-
state = state if batched else state.unsqueeze(0)
|
|
97
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
98
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
102
99
|
if self.is_continuous:
|
|
103
100
|
action_mean, action_log_std = self.actor(state)
|
|
104
101
|
action = torch.tanh(Normal(action_mean, action_log_std.exp()).sample())
|
|
105
102
|
else:
|
|
106
103
|
action_logits = self.actor(state)
|
|
107
104
|
action = Categorical(logits=action_logits).sample()
|
|
108
|
-
action = action if batched else action[0]
|
|
109
105
|
|
|
110
106
|
if self.is_continuous:
|
|
111
107
|
env_action = action * self.action_scale + self.action_bias
|
|
112
108
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
113
109
|
else:
|
|
114
|
-
action = action.cpu().numpy()
|
|
110
|
+
action = action.cpu().numpy()
|
|
115
111
|
env_action = action
|
|
116
112
|
|
|
117
|
-
return {"
|
|
113
|
+
return {"action": action, "env_action": env_action, "extra_data": {}}
|
|
@@ -74,21 +74,18 @@ class Policy:
|
|
|
74
74
|
return action.squeeze(0).cpu().numpy()
|
|
75
75
|
|
|
76
76
|
@torch.no_grad()
|
|
77
|
-
def sample_action(self,
|
|
78
|
-
"""
|
|
77
|
+
def sample_action(self, observations):
|
|
78
|
+
"""Sample noisy normalized actions for a batch of observations."""
|
|
79
79
|
if self.random_warmup and self.version == 0:
|
|
80
|
-
scale = self.action_scale
|
|
81
|
-
scale = scale.expand(len(observation), -1) if batched else scale
|
|
80
|
+
scale = self.action_scale.expand(len(observations), -1)
|
|
82
81
|
action = torch.empty_like(scale).uniform_(-1, 1)
|
|
83
82
|
else:
|
|
84
|
-
|
|
85
|
-
state = torch.as_tensor(
|
|
86
|
-
state = state if batched else state.unsqueeze(0)
|
|
83
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
84
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
87
85
|
action = self.actor(state)
|
|
88
86
|
noise = torch.randn_like(action) * self.action_noise
|
|
89
87
|
action = torch.clamp(action + noise, -1, 1)
|
|
90
|
-
action = action if batched else action[0]
|
|
91
88
|
|
|
92
89
|
env_action = action * self.action_scale + self.action_bias
|
|
93
90
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
94
|
-
return {"
|
|
91
|
+
return {"action": action, "env_action": env_action, "extra_data": {}}
|
|
@@ -78,20 +78,17 @@ class Policy:
|
|
|
78
78
|
return action.squeeze(0).cpu().numpy()
|
|
79
79
|
|
|
80
80
|
@torch.no_grad()
|
|
81
|
-
def sample_action(self,
|
|
82
|
-
"""
|
|
81
|
+
def sample_action(self, observations):
|
|
82
|
+
"""Sample stochastic normalized actions for a batch of observations."""
|
|
83
83
|
if self.random_warmup and self.version == 0:
|
|
84
|
-
scale = self.action_scale
|
|
85
|
-
scale = scale.expand(len(observation), -1) if batched else scale
|
|
84
|
+
scale = self.action_scale.expand(len(observations), -1)
|
|
86
85
|
action = torch.empty_like(scale).uniform_(-1, 1)
|
|
87
86
|
else:
|
|
88
|
-
|
|
89
|
-
state = torch.as_tensor(
|
|
90
|
-
state = state if batched else state.unsqueeze(0)
|
|
87
|
+
observations = np.asarray(observations, dtype=np.float32)
|
|
88
|
+
state = torch.as_tensor(observations, device=self.device)
|
|
91
89
|
action_mean, action_log_std = self.actor(state)
|
|
92
90
|
action = torch.tanh(Normal(action_mean, action_log_std.exp()).sample())
|
|
93
|
-
action = action if batched else action[0]
|
|
94
91
|
|
|
95
92
|
env_action = action * self.action_scale + self.action_bias
|
|
96
93
|
action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
|
|
97
|
-
return {"
|
|
94
|
+
return {"action": action, "env_action": env_action, "extra_data": {}}
|
{omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/prioritized_replay_buffer.py
RENAMED
|
@@ -58,33 +58,54 @@ class PrioritizedReplayBuffer:
|
|
|
58
58
|
self.next_state_memory = np.zeros((max_size, *state_dim), dtype=state_dtype)
|
|
59
59
|
self.terminal_memory = np.zeros(max_size, dtype=np.bool_)
|
|
60
60
|
self.priority_memory = np.zeros(max_size, dtype=np.float32)
|
|
61
|
-
|
|
62
|
-
def put(self, state, action, reward, next_state, terminal):
|
|
63
|
-
"""Store one transition into the replay buffer."""
|
|
64
|
-
index = self.collected_size % self.max_size
|
|
65
|
-
|
|
66
|
-
self.state_memory[index] = state
|
|
67
|
-
self.action_memory[index] = action
|
|
68
|
-
self.reward_memory[index] = reward
|
|
69
|
-
self.next_state_memory[index] = next_state
|
|
70
|
-
self.terminal_memory[index] = terminal
|
|
71
|
-
self.priority_memory[index] = self.max_priority
|
|
72
|
-
|
|
73
|
-
self.collected_size += 1
|
|
74
|
-
self.available_size = min(self.max_size, self.collected_size)
|
|
61
|
+
self.extra_data_memory = {}
|
|
75
62
|
|
|
76
63
|
def add(self, rollout):
|
|
77
|
-
"""Store
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
64
|
+
"""Store a rollout with leading [num_envs, num_steps] dimensions."""
|
|
65
|
+
count = rollout["rewards"].size
|
|
66
|
+
size = min(count, self.max_size)
|
|
67
|
+
offset = count - size
|
|
68
|
+
start = (self.collected_size + offset) % self.max_size
|
|
69
|
+
end = min(start + size, self.max_size)
|
|
70
|
+
first_size = end - start
|
|
71
|
+
remaining_size = size - first_size
|
|
72
|
+
|
|
73
|
+
# Initialize extra data storage.
|
|
74
|
+
extra_data = rollout["extra_data"]
|
|
75
|
+
if self.collected_size == 0:
|
|
76
|
+
for name, values in extra_data.items():
|
|
77
|
+
memory = np.empty((self.max_size, *values.shape[2:]), dtype=values.dtype)
|
|
78
|
+
self.extra_data_memory[name] = memory
|
|
79
|
+
|
|
80
|
+
state_shape = self.state_memory.shape[1:]
|
|
81
|
+
action_shape = self.action_memory.shape[1:]
|
|
82
|
+
states = rollout["states"].reshape(count, *state_shape)[offset:]
|
|
83
|
+
actions = rollout["action"].reshape(count, *action_shape)[offset:]
|
|
84
|
+
rewards = rollout["rewards"].reshape(count)[offset:]
|
|
85
|
+
next_states = rollout["next_states"].reshape(count, *state_shape)[offset:]
|
|
86
|
+
terminals = rollout["terminals"].reshape(count)[offset:]
|
|
87
|
+
|
|
88
|
+
self.state_memory[start:end] = states[:first_size]
|
|
89
|
+
self.action_memory[start:end] = actions[:first_size]
|
|
90
|
+
self.reward_memory[start:end] = rewards[:first_size]
|
|
91
|
+
self.next_state_memory[start:end] = next_states[:first_size]
|
|
92
|
+
self.terminal_memory[start:end] = terminals[:first_size]
|
|
93
|
+
self.priority_memory[start:end] = self.max_priority
|
|
94
|
+
|
|
95
|
+
self.state_memory[:remaining_size] = states[first_size:]
|
|
96
|
+
self.action_memory[:remaining_size] = actions[first_size:]
|
|
97
|
+
self.reward_memory[:remaining_size] = rewards[first_size:]
|
|
98
|
+
self.next_state_memory[:remaining_size] = next_states[first_size:]
|
|
99
|
+
self.terminal_memory[:remaining_size] = terminals[first_size:]
|
|
100
|
+
self.priority_memory[:remaining_size] = self.max_priority
|
|
101
|
+
for name, values in extra_data.items():
|
|
102
|
+
values = values.reshape(count, *values.shape[2:])[offset:]
|
|
103
|
+
memory = self.extra_data_memory[name]
|
|
104
|
+
memory[start:end] = values[:first_size]
|
|
105
|
+
memory[:remaining_size] = values[first_size:]
|
|
106
|
+
|
|
107
|
+
self.collected_size += count
|
|
108
|
+
self.available_size = min(self.max_size, self.collected_size)
|
|
88
109
|
|
|
89
110
|
def get_batch(self, rank, world_size):
|
|
90
111
|
"""Sample a rank-specific mini-batch from the replay buffer."""
|
|
@@ -107,7 +128,11 @@ class PrioritizedReplayBuffer:
|
|
|
107
128
|
max_weight = (self.available_size * probs.min()) ** (-self.beta)
|
|
108
129
|
weights /= max_weight
|
|
109
130
|
|
|
110
|
-
|
|
131
|
+
extra_data = {}
|
|
132
|
+
for name, memory in self.extra_data_memory.items():
|
|
133
|
+
extra_data[name] = memory[indices]
|
|
134
|
+
|
|
135
|
+
batch = {
|
|
111
136
|
"states": self.state_memory[indices],
|
|
112
137
|
"actions": self.action_memory[indices],
|
|
113
138
|
"rewards": self.reward_memory[indices],
|
|
@@ -115,7 +140,9 @@ class PrioritizedReplayBuffer:
|
|
|
115
140
|
"terminals": self.terminal_memory[indices],
|
|
116
141
|
"indices": indices,
|
|
117
142
|
"weights": weights.astype(np.float32),
|
|
143
|
+
**extra_data,
|
|
118
144
|
}
|
|
145
|
+
return batch
|
|
119
146
|
|
|
120
147
|
def on_learn(self, train_infos):
|
|
121
148
|
"""Update internal state after the learner finishes one training step."""
|
|
@@ -149,14 +176,6 @@ class PrioritizedReplayBuffer:
|
|
|
149
176
|
"""Return True because batches can be sampled before the last update."""
|
|
150
177
|
return True
|
|
151
178
|
|
|
152
|
-
def _clear(self):
|
|
153
|
-
"""Clear all stored transitions in the replay buffer."""
|
|
154
|
-
self.available_size = 0
|
|
155
|
-
self.collected_size = 0
|
|
156
|
-
self.replayed_size = 0
|
|
157
|
-
self.beta = self.beta_start
|
|
158
|
-
self.max_priority = 1.0
|
|
159
|
-
|
|
160
179
|
def stats(self):
|
|
161
180
|
"""Return statistics describing the current state of the replay buffer."""
|
|
162
181
|
return {
|
|
@@ -45,32 +45,52 @@ class ReplayBuffer:
|
|
|
45
45
|
self.reward_memory = np.zeros(max_size, dtype=np.float32)
|
|
46
46
|
self.next_state_memory = np.zeros((max_size, *state_dim), dtype=state_dtype)
|
|
47
47
|
self.terminal_memory = np.zeros(max_size, dtype=np.bool_)
|
|
48
|
-
|
|
49
|
-
def put(self, state, action, reward, next_state, terminal):
|
|
50
|
-
"""Store one transition into the replay buffer."""
|
|
51
|
-
index = self.collected_size % self.max_size
|
|
52
|
-
|
|
53
|
-
self.state_memory[index] = state
|
|
54
|
-
self.action_memory[index] = action
|
|
55
|
-
self.reward_memory[index] = reward
|
|
56
|
-
self.next_state_memory[index] = next_state
|
|
57
|
-
self.terminal_memory[index] = terminal
|
|
58
|
-
|
|
59
|
-
self.collected_size += 1
|
|
60
|
-
self.available_size = min(self.max_size, self.collected_size)
|
|
48
|
+
self.extra_data_memory = {}
|
|
61
49
|
|
|
62
50
|
def add(self, rollout):
|
|
63
|
-
"""Store
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
51
|
+
"""Store a rollout with leading [num_envs, num_steps] dimensions."""
|
|
52
|
+
count = rollout["rewards"].size
|
|
53
|
+
size = min(count, self.max_size)
|
|
54
|
+
offset = count - size
|
|
55
|
+
start = (self.collected_size + offset) % self.max_size
|
|
56
|
+
end = min(start + size, self.max_size)
|
|
57
|
+
first_size = end - start
|
|
58
|
+
remaining_size = size - first_size
|
|
59
|
+
|
|
60
|
+
# Initialize extra data storage.
|
|
61
|
+
extra_data = rollout["extra_data"]
|
|
62
|
+
if self.collected_size == 0:
|
|
63
|
+
for name, values in extra_data.items():
|
|
64
|
+
memory = np.empty((self.max_size, *values.shape[2:]), dtype=values.dtype)
|
|
65
|
+
self.extra_data_memory[name] = memory
|
|
66
|
+
|
|
67
|
+
state_shape = self.state_memory.shape[1:]
|
|
68
|
+
action_shape = self.action_memory.shape[1:]
|
|
69
|
+
states = rollout["states"].reshape(count, *state_shape)[offset:]
|
|
70
|
+
actions = rollout["action"].reshape(count, *action_shape)[offset:]
|
|
71
|
+
rewards = rollout["rewards"].reshape(count)[offset:]
|
|
72
|
+
next_states = rollout["next_states"].reshape(count, *state_shape)[offset:]
|
|
73
|
+
terminals = rollout["terminals"].reshape(count)[offset:]
|
|
74
|
+
|
|
75
|
+
self.state_memory[start:end] = states[:first_size]
|
|
76
|
+
self.action_memory[start:end] = actions[:first_size]
|
|
77
|
+
self.reward_memory[start:end] = rewards[:first_size]
|
|
78
|
+
self.next_state_memory[start:end] = next_states[:first_size]
|
|
79
|
+
self.terminal_memory[start:end] = terminals[:first_size]
|
|
80
|
+
|
|
81
|
+
self.state_memory[:remaining_size] = states[first_size:]
|
|
82
|
+
self.action_memory[:remaining_size] = actions[first_size:]
|
|
83
|
+
self.reward_memory[:remaining_size] = rewards[first_size:]
|
|
84
|
+
self.next_state_memory[:remaining_size] = next_states[first_size:]
|
|
85
|
+
self.terminal_memory[:remaining_size] = terminals[first_size:]
|
|
86
|
+
for name, values in extra_data.items():
|
|
87
|
+
values = values.reshape(count, *values.shape[2:])[offset:]
|
|
88
|
+
memory = self.extra_data_memory[name]
|
|
89
|
+
memory[start:end] = values[:first_size]
|
|
90
|
+
memory[:remaining_size] = values[first_size:]
|
|
91
|
+
|
|
92
|
+
self.collected_size += count
|
|
93
|
+
self.available_size = min(self.max_size, self.collected_size)
|
|
74
94
|
|
|
75
95
|
def get_batch(self, rank, world_size):
|
|
76
96
|
"""Sample a rank-specific mini-batch from the replay buffer."""
|
|
@@ -84,13 +104,19 @@ class ReplayBuffer:
|
|
|
84
104
|
|
|
85
105
|
indices = np.random.choice(self.available_size, size=size, replace=False)
|
|
86
106
|
|
|
87
|
-
|
|
107
|
+
extra_data = {}
|
|
108
|
+
for name, memory in self.extra_data_memory.items():
|
|
109
|
+
extra_data[name] = memory[indices]
|
|
110
|
+
|
|
111
|
+
batch = {
|
|
88
112
|
"states": self.state_memory[indices],
|
|
89
113
|
"actions": self.action_memory[indices],
|
|
90
114
|
"rewards": self.reward_memory[indices],
|
|
91
115
|
"next_states": self.next_state_memory[indices],
|
|
92
116
|
"terminals": self.terminal_memory[indices],
|
|
117
|
+
**extra_data,
|
|
93
118
|
}
|
|
119
|
+
return batch
|
|
94
120
|
|
|
95
121
|
def on_learn(self, train_infos):
|
|
96
122
|
"""Update internal state after the learner finishes one training step."""
|
|
@@ -106,12 +132,6 @@ class ReplayBuffer:
|
|
|
106
132
|
"""Return True because batches can be sampled before the last update."""
|
|
107
133
|
return True
|
|
108
134
|
|
|
109
|
-
def _clear(self):
|
|
110
|
-
"""Clear all stored transitions in the replay buffer."""
|
|
111
|
-
self.available_size = 0
|
|
112
|
-
self.collected_size = 0
|
|
113
|
-
self.replayed_size = 0
|
|
114
|
-
|
|
115
135
|
def stats(self):
|
|
116
136
|
"""Return statistics describing the current state of the replay buffer."""
|
|
117
137
|
return {
|