omnirl-framework 0.3.0__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {omnirl_framework-0.3.0/omnirl_framework.egg-info → omnirl_framework-0.3.2}/PKG-INFO +2 -2
  2. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/README.md +1 -1
  3. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/policy.py +6 -8
  4. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/policy.py +5 -7
  5. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/policy.py +6 -9
  6. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/policy.py +5 -7
  7. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/policy.py +5 -7
  8. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/policy.py +5 -7
  9. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/policy.py +8 -12
  10. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/policy.py +5 -7
  11. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/policy.py +8 -12
  12. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/policy.py +5 -7
  13. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/policy.py +8 -12
  14. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/policy.py +6 -9
  15. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/policy.py +6 -9
  16. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/prioritized_replay_buffer.py +53 -34
  17. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/replay_buffer.py +51 -31
  18. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/rollout_buffer.py +32 -45
  19. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/collection.py +18 -13
  20. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/sampler.py +3 -4
  21. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2/omnirl_framework.egg-info}/PKG-INFO +2 -2
  22. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/pyproject.toml +1 -1
  23. omnirl_framework-0.3.2/tests/test_gymnasium.py +127 -0
  24. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_integration.py +19 -2
  25. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_runtime.py +55 -39
  26. omnirl_framework-0.3.0/tests/test_gymnasium.py +0 -45
  27. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/LICENSE +0 -0
  28. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/__init__.py +0 -0
  29. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/__init__.py +0 -0
  30. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/__init__.py +0 -0
  31. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/a2c/learner.py +0 -0
  32. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/__init__.py +0 -0
  33. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/c51_dqn/learner.py +0 -0
  34. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/__init__.py +0 -0
  35. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddpg/learner.py +0 -0
  36. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/__init__.py +0 -0
  37. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ddqn/learner.py +0 -0
  38. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/__init__.py +0 -0
  39. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dqn/learner.py +0 -0
  40. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/__init__.py +0 -0
  41. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/dueling_dqn/learner.py +0 -0
  42. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/__init__.py +0 -0
  43. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/impala/learner.py +0 -0
  44. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/__init__.py +0 -0
  45. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/per_dqn/learner.py +0 -0
  46. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/__init__.py +0 -0
  47. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/ppo/learner.py +0 -0
  48. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/__init__.py +0 -0
  49. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/qr_dqn/learner.py +0 -0
  50. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/__init__.py +0 -0
  51. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/sac/learner.py +0 -0
  52. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/__init__.py +0 -0
  53. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/td3/learner.py +0 -0
  54. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/__init__.py +0 -0
  55. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/algorithms/tqc/learner.py +0 -0
  56. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/buffers/__init__.py +0 -0
  57. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/__init__.py +0 -0
  58. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/__init__.py +0 -0
  59. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/adapter.py +0 -0
  60. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/builder.py +0 -0
  61. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/evaluation.py +0 -0
  62. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/envs/gymnasium/spec.py +0 -0
  63. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/__init__.py +0 -0
  64. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/actors.py +0 -0
  65. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/__init__.py +0 -0
  66. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/cnn.py +0 -0
  67. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/mlp.py +0 -0
  68. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/base/resnet.py +0 -0
  69. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/networks/critics.py +0 -0
  70. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/__init__.py +0 -0
  71. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/evaluator.py +0 -0
  72. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/file_writer.py +0 -0
  73. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/logger.py +0 -0
  74. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/runner.py +0 -0
  75. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/runtime/weight_store.py +0 -0
  76. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/utils/__init__.py +0 -0
  77. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl/utils/seed.py +0 -0
  78. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/SOURCES.txt +0 -0
  79. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/dependency_links.txt +0 -0
  80. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/requires.txt +0 -0
  81. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/omnirl_framework.egg-info/top_level.txt +0 -0
  82. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/requirements/base.txt +0 -0
  83. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/setup.cfg +0 -0
  84. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_networks.py +0 -0
  85. {omnirl_framework-0.3.0 → omnirl_framework-0.3.2}/tests/test_structure.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: omnirl-framework
3
- Version: 0.3.0
3
+ Version: 0.3.2
4
4
  Summary: A distributed reinforcement learning framework.
5
5
  Author-email: Huang Chenbiao <a1057951125@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -197,7 +197,7 @@ __all__ = ["Learner", "Memory", "Policy"]
197
197
  The implementation should provide:
198
198
 
199
199
  - `Learner`: a Ray actor that implements learning, checkpointing, and versioned weight access. Existing learners can be used as interface examples.
200
- - `Policy`: a local inference class with deterministic `choose_action` and exploratory `sample_action` methods.
200
+ - `Policy`: a local inference class with deterministic `choose_action(observation)` for a single observation and exploratory `sample_action(observations)` for a batch. Sampling inputs and outputs must retain the leading batch dimension, including when the batch size is 1.
201
201
  - `Memory`: one of the buffers in `omnirl.buffers`, or a compatible new Ray actor.
202
202
  - Network definitions under `omnirl.networks` when the algorithm needs a new architecture.
203
203
  - An agent YAML whose `agent.module` points to `omnirl.algorithms.<name>` and whose `modules` section supplies the constructor parameters.
@@ -169,7 +169,7 @@ __all__ = ["Learner", "Memory", "Policy"]
169
169
  The implementation should provide:
170
170
 
171
171
  - `Learner`: a Ray actor that implements learning, checkpointing, and versioned weight access. Existing learners can be used as interface examples.
172
- - `Policy`: a local inference class with deterministic `choose_action` and exploratory `sample_action` methods.
172
+ - `Policy`: a local inference class with deterministic `choose_action(observation)` for a single observation and exploratory `sample_action(observations)` for a batch. Sampling inputs and outputs must retain the leading batch dimension, including when the batch size is 1.
173
173
  - `Memory`: one of the buffers in `omnirl.buffers`, or a compatible new Ray actor.
174
174
  - Network definitions under `omnirl.networks` when the algorithm needs a new architecture.
175
175
  - An agent YAML whose `agent.module` points to `omnirl.algorithms.<name>` and whose `modules` section supplies the constructor parameters.
@@ -81,22 +81,20 @@ class Policy:
81
81
  return int(action.item())
82
82
 
83
83
  @torch.no_grad()
84
- def sample_action(self, observation, batched=False):
85
- """Select a probabilistic action during training."""
86
- observation = np.asarray(observation, dtype=np.float32)
87
- state = torch.as_tensor(observation, device=self.device)
88
- state = state if batched else state.unsqueeze(0)
84
+ def sample_action(self, observations):
85
+ """Sample actions for observations shaped [batch, *state_shape]."""
86
+ observations = np.asarray(observations, dtype=np.float32)
87
+ state = torch.as_tensor(observations, device=self.device)
89
88
 
90
89
  if self.is_continuous:
91
90
  action_mean, action_log_std = self.actor(state)
92
91
  action = Normal(action_mean, action_log_std.exp()).sample()
93
- action = action if batched else action[0]
94
92
  env_action = torch.tanh(action) * self.action_scale + self.action_bias
95
93
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
96
94
  else:
97
95
  action_logits = self.actor(state)
98
96
  action = Categorical(logits=action_logits).sample()
99
- action = action.cpu().numpy() if batched else int(action.item())
97
+ action = action.cpu().numpy()
100
98
  env_action = action
101
99
 
102
- return {"env_action": env_action, "data": {"action": action}}
100
+ return {"action": action, "env_action": env_action, "extra_data": {}}
@@ -84,11 +84,10 @@ class Policy:
84
84
  return int(torch.argmax(q_values, dim=-1).item())
85
85
 
86
86
  @torch.no_grad()
87
- def sample_action(self, observation, batched=False):
88
- """Select epsilon-greedy actions, optionally for a batch of envs."""
89
- observation = np.asarray(observation, dtype=np.float32)
90
- state = torch.as_tensor(observation, device=self.device)
91
- state = state if batched else state.unsqueeze(0)
87
+ def sample_action(self, observations):
88
+ """Select epsilon-greedy actions for a batch of observations."""
89
+ observations = np.asarray(observations, dtype=np.float32)
90
+ state = torch.as_tensor(observations, device=self.device)
92
91
 
93
92
  action = torch.randint(self.action_dim, (len(state),)).numpy()
94
93
  if not (self.random_warmup and self.version == 0):
@@ -100,5 +99,4 @@ class Policy:
100
99
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
101
100
  action = np.where(random_mask, action, greedy_action)
102
101
 
103
- action = action if batched else int(action[0])
104
- return {"env_action": action, "data": {"action": action}}
102
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -74,21 +74,18 @@ class Policy:
74
74
  return action.squeeze(0).cpu().numpy()
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select a noisy normalized action during training."""
77
+ def sample_action(self, observations):
78
+ """Sample noisy normalized actions for a batch of observations."""
79
79
  if self.random_warmup and self.version == 0:
80
- scale = self.action_scale
81
- scale = scale.expand(len(observation), -1) if batched else scale
80
+ scale = self.action_scale.expand(len(observations), -1)
82
81
  action = torch.empty_like(scale).uniform_(-1, 1)
83
82
  else:
84
- observation = np.asarray(observation, dtype=np.float32)
85
- state = torch.as_tensor(observation, device=self.device)
86
- state = state if batched else state.unsqueeze(0)
83
+ observations = np.asarray(observations, dtype=np.float32)
84
+ state = torch.as_tensor(observations, device=self.device)
87
85
  action = self.actor(state)
88
86
  noise = torch.randn_like(action) * self.action_noise
89
87
  action = torch.clamp(action + noise, -1, 1)
90
- action = action if batched else action[0]
91
88
 
92
89
  env_action = action * self.action_scale + self.action_bias
93
90
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
94
- return {"env_action": env_action, "data": {"action": action}}
91
+ return {"action": action, "env_action": env_action, "extra_data": {}}
@@ -74,11 +74,10 @@ class Policy:
74
74
  return int(torch.argmax(q_values, dim=-1).item())
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select epsilon-greedy actions, optionally for a batch of envs."""
79
- observation = np.asarray(observation, dtype=np.float32)
80
- state = torch.as_tensor(observation, device=self.device)
81
- state = state if batched else state.unsqueeze(0)
77
+ def sample_action(self, observations):
78
+ """Select epsilon-greedy actions for a batch of observations."""
79
+ observations = np.asarray(observations, dtype=np.float32)
80
+ state = torch.as_tensor(observations, device=self.device)
82
81
 
83
82
  action = torch.randint(self.action_dim, (len(state),)).numpy()
84
83
  if not (self.random_warmup and self.version == 0):
@@ -88,5 +87,4 @@ class Policy:
88
87
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
89
88
  action = np.where(random_mask, action, greedy_action)
90
89
 
91
- action = action if batched else int(action[0])
92
- return {"env_action": action, "data": {"action": action}}
90
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -74,11 +74,10 @@ class Policy:
74
74
  return int(torch.argmax(q_values, dim=-1).item())
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select epsilon-greedy actions, optionally for a batch of envs."""
79
- observation = np.asarray(observation, dtype=np.float32)
80
- state = torch.as_tensor(observation, device=self.device)
81
- state = state if batched else state.unsqueeze(0)
77
+ def sample_action(self, observations):
78
+ """Select epsilon-greedy actions for a batch of observations."""
79
+ observations = np.asarray(observations, dtype=np.float32)
80
+ state = torch.as_tensor(observations, device=self.device)
82
81
 
83
82
  action = torch.randint(self.action_dim, (len(state),)).numpy()
84
83
  if not (self.random_warmup and self.version == 0):
@@ -88,5 +87,4 @@ class Policy:
88
87
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
89
88
  action = np.where(random_mask, action, greedy_action)
90
89
 
91
- action = action if batched else int(action[0])
92
- return {"env_action": action, "data": {"action": action}}
90
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -74,11 +74,10 @@ class Policy:
74
74
  return int(torch.argmax(q_values, dim=-1).item())
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select epsilon-greedy actions, optionally for a batch of envs."""
79
- observation = np.asarray(observation, dtype=np.float32)
80
- state = torch.as_tensor(observation, device=self.device)
81
- state = state if batched else state.unsqueeze(0)
77
+ def sample_action(self, observations):
78
+ """Select epsilon-greedy actions for a batch of observations."""
79
+ observations = np.asarray(observations, dtype=np.float32)
80
+ state = torch.as_tensor(observations, device=self.device)
82
81
 
83
82
  action = torch.randint(self.action_dim, (len(state),)).numpy()
84
83
  if not (self.random_warmup and self.version == 0):
@@ -88,5 +87,4 @@ class Policy:
88
87
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
89
88
  action = np.where(random_mask, action, greedy_action)
90
89
 
91
- action = action if batched else int(action[0])
92
- return {"env_action": action, "data": {"action": action}}
90
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -81,18 +81,16 @@ class Policy:
81
81
  return int(action.item())
82
82
 
83
83
  @torch.no_grad()
84
- def sample_action(self, observation, batched=False):
85
- """Select a probabilistic action and its log probability."""
86
- observation = np.asarray(observation, dtype=np.float32)
87
- state = torch.as_tensor(observation, device=self.device)
88
- state = state if batched else state.unsqueeze(0)
84
+ def sample_action(self, observations):
85
+ """Sample actions and log probabilities for a batch of observations."""
86
+ observations = np.asarray(observations, dtype=np.float32)
87
+ state = torch.as_tensor(observations, device=self.device)
89
88
 
90
89
  if self.is_continuous:
91
90
  action_mean, action_log_std = self.actor(state)
92
91
  dist = Normal(action_mean, action_log_std.exp())
93
92
  action = dist.sample()
94
93
  log_prob = dist.log_prob(action).sum(dim=-1)
95
- action = action if batched else action[0]
96
94
  env_action = torch.tanh(action) * self.action_scale + self.action_bias
97
95
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
98
96
  else:
@@ -100,11 +98,9 @@ class Policy:
100
98
  dist = Categorical(logits=action_logits)
101
99
  action = dist.sample()
102
100
  log_prob = dist.log_prob(action)
103
- action = action.cpu().numpy() if batched else int(action.item())
101
+ action = action.cpu().numpy()
104
102
  env_action = action
105
103
 
106
- log_prob = log_prob.cpu().numpy() if batched else log_prob.item()
107
- return {
108
- "env_action": env_action,
109
- "data": {"action": action, "log_prob": log_prob},
110
- }
104
+ log_prob = log_prob.cpu().numpy()
105
+ extra_data = {"log_probs": log_prob}
106
+ return {"action": action, "env_action": env_action, "extra_data": extra_data}
@@ -74,11 +74,10 @@ class Policy:
74
74
  return int(torch.argmax(q_values, dim=-1).item())
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select epsilon-greedy actions, optionally for a batch of envs."""
79
- observation = np.asarray(observation, dtype=np.float32)
80
- state = torch.as_tensor(observation, device=self.device)
81
- state = state if batched else state.unsqueeze(0)
77
+ def sample_action(self, observations):
78
+ """Select epsilon-greedy actions for a batch of observations."""
79
+ observations = np.asarray(observations, dtype=np.float32)
80
+ state = torch.as_tensor(observations, device=self.device)
82
81
 
83
82
  action = torch.randint(self.action_dim, (len(state),)).numpy()
84
83
  if not (self.random_warmup and self.version == 0):
@@ -88,5 +87,4 @@ class Policy:
88
87
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
89
88
  action = np.where(random_mask, action, greedy_action)
90
89
 
91
- action = action if batched else int(action[0])
92
- return {"env_action": action, "data": {"action": action}}
90
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -81,18 +81,16 @@ class Policy:
81
81
  return int(action.item())
82
82
 
83
83
  @torch.no_grad()
84
- def sample_action(self, observation, batched=False):
85
- """Select a probabilistic action and its log probability."""
86
- observation = np.asarray(observation, dtype=np.float32)
87
- state = torch.as_tensor(observation, device=self.device)
88
- state = state if batched else state.unsqueeze(0)
84
+ def sample_action(self, observations):
85
+ """Sample actions and log probabilities for a batch of observations."""
86
+ observations = np.asarray(observations, dtype=np.float32)
87
+ state = torch.as_tensor(observations, device=self.device)
89
88
 
90
89
  if self.is_continuous:
91
90
  action_mean, action_log_std = self.actor(state)
92
91
  dist = Normal(action_mean, action_log_std.exp())
93
92
  action = dist.sample()
94
93
  log_prob = dist.log_prob(action).sum(dim=-1)
95
- action = action if batched else action[0]
96
94
  env_action = torch.tanh(action) * self.action_scale + self.action_bias
97
95
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
98
96
  else:
@@ -100,11 +98,9 @@ class Policy:
100
98
  dist = Categorical(logits=action_logits)
101
99
  action = dist.sample()
102
100
  log_prob = dist.log_prob(action)
103
- action = action.cpu().numpy() if batched else int(action.item())
101
+ action = action.cpu().numpy()
104
102
  env_action = action
105
103
 
106
- log_prob = log_prob.cpu().numpy() if batched else log_prob.item()
107
- return {
108
- "env_action": env_action,
109
- "data": {"action": action, "log_prob": log_prob},
110
- }
104
+ log_prob = log_prob.cpu().numpy()
105
+ extra_data = {"log_probs": log_prob}
106
+ return {"action": action, "env_action": env_action, "extra_data": extra_data}
@@ -78,11 +78,10 @@ class Policy:
78
78
  return int(torch.argmax(q_values, dim=-1).item())
79
79
 
80
80
  @torch.no_grad()
81
- def sample_action(self, observation, batched=False):
82
- """Select epsilon-greedy actions, optionally for a batch of envs."""
83
- observation = np.asarray(observation, dtype=np.float32)
84
- state = torch.as_tensor(observation, device=self.device)
85
- state = state if batched else state.unsqueeze(0)
81
+ def sample_action(self, observations):
82
+ """Select epsilon-greedy actions for a batch of observations."""
83
+ observations = np.asarray(observations, dtype=np.float32)
84
+ state = torch.as_tensor(observations, device=self.device)
86
85
 
87
86
  action = torch.randint(self.action_dim, (len(state),)).numpy()
88
87
  if not (self.random_warmup and self.version == 0):
@@ -92,5 +91,4 @@ class Policy:
92
91
  greedy_action = torch.argmax(q_values, dim=-1).cpu().numpy()
93
92
  action = np.where(random_mask, action, greedy_action)
94
93
 
95
- action = action if batched else int(action[0])
96
- return {"env_action": action, "data": {"action": action}}
94
+ return {"action": action, "env_action": action, "extra_data": {}}
@@ -85,33 +85,29 @@ class Policy:
85
85
  return int(action.item())
86
86
 
87
87
  @torch.no_grad()
88
- def sample_action(self, observation, batched=False):
89
- """Select a stochastic action during training."""
88
+ def sample_action(self, observations):
89
+ """Sample stochastic actions for a batch of observations."""
90
90
  if self.random_warmup and self.version == 0:
91
91
  if self.is_continuous:
92
- scale = self.action_scale
93
- scale = scale.expand(len(observation), -1) if batched else scale
92
+ scale = self.action_scale.expand(len(observations), -1)
94
93
  action = torch.empty_like(scale).uniform_(-1, 1)
95
94
  else:
96
- size = len(observation) if batched else 1
97
- action = torch.randint(self.action_dim, (size,))
95
+ action = torch.randint(self.action_dim, (len(observations),))
98
96
  else:
99
- observation = np.asarray(observation, dtype=np.float32)
100
- state = torch.as_tensor(observation, device=self.device)
101
- state = state if batched else state.unsqueeze(0)
97
+ observations = np.asarray(observations, dtype=np.float32)
98
+ state = torch.as_tensor(observations, device=self.device)
102
99
  if self.is_continuous:
103
100
  action_mean, action_log_std = self.actor(state)
104
101
  action = torch.tanh(Normal(action_mean, action_log_std.exp()).sample())
105
102
  else:
106
103
  action_logits = self.actor(state)
107
104
  action = Categorical(logits=action_logits).sample()
108
- action = action if batched else action[0]
109
105
 
110
106
  if self.is_continuous:
111
107
  env_action = action * self.action_scale + self.action_bias
112
108
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
113
109
  else:
114
- action = action.cpu().numpy() if batched else int(action.item())
110
+ action = action.cpu().numpy()
115
111
  env_action = action
116
112
 
117
- return {"env_action": env_action, "data": {"action": action}}
113
+ return {"action": action, "env_action": env_action, "extra_data": {}}
@@ -74,21 +74,18 @@ class Policy:
74
74
  return action.squeeze(0).cpu().numpy()
75
75
 
76
76
  @torch.no_grad()
77
- def sample_action(self, observation, batched=False):
78
- """Select a noisy normalized action during training."""
77
+ def sample_action(self, observations):
78
+ """Sample noisy normalized actions for a batch of observations."""
79
79
  if self.random_warmup and self.version == 0:
80
- scale = self.action_scale
81
- scale = scale.expand(len(observation), -1) if batched else scale
80
+ scale = self.action_scale.expand(len(observations), -1)
82
81
  action = torch.empty_like(scale).uniform_(-1, 1)
83
82
  else:
84
- observation = np.asarray(observation, dtype=np.float32)
85
- state = torch.as_tensor(observation, device=self.device)
86
- state = state if batched else state.unsqueeze(0)
83
+ observations = np.asarray(observations, dtype=np.float32)
84
+ state = torch.as_tensor(observations, device=self.device)
87
85
  action = self.actor(state)
88
86
  noise = torch.randn_like(action) * self.action_noise
89
87
  action = torch.clamp(action + noise, -1, 1)
90
- action = action if batched else action[0]
91
88
 
92
89
  env_action = action * self.action_scale + self.action_bias
93
90
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
94
- return {"env_action": env_action, "data": {"action": action}}
91
+ return {"action": action, "env_action": env_action, "extra_data": {}}
@@ -78,20 +78,17 @@ class Policy:
78
78
  return action.squeeze(0).cpu().numpy()
79
79
 
80
80
  @torch.no_grad()
81
- def sample_action(self, observation, batched=False):
82
- """Select a stochastic normalized action during training."""
81
+ def sample_action(self, observations):
82
+ """Sample stochastic normalized actions for a batch of observations."""
83
83
  if self.random_warmup and self.version == 0:
84
- scale = self.action_scale
85
- scale = scale.expand(len(observation), -1) if batched else scale
84
+ scale = self.action_scale.expand(len(observations), -1)
86
85
  action = torch.empty_like(scale).uniform_(-1, 1)
87
86
  else:
88
- observation = np.asarray(observation, dtype=np.float32)
89
- state = torch.as_tensor(observation, device=self.device)
90
- state = state if batched else state.unsqueeze(0)
87
+ observations = np.asarray(observations, dtype=np.float32)
88
+ state = torch.as_tensor(observations, device=self.device)
91
89
  action_mean, action_log_std = self.actor(state)
92
90
  action = torch.tanh(Normal(action_mean, action_log_std.exp()).sample())
93
- action = action if batched else action[0]
94
91
 
95
92
  env_action = action * self.action_scale + self.action_bias
96
93
  action, env_action = action.cpu().numpy(), env_action.cpu().numpy()
97
- return {"env_action": env_action, "data": {"action": action}}
94
+ return {"action": action, "env_action": env_action, "extra_data": {}}
@@ -58,33 +58,54 @@ class PrioritizedReplayBuffer:
58
58
  self.next_state_memory = np.zeros((max_size, *state_dim), dtype=state_dtype)
59
59
  self.terminal_memory = np.zeros(max_size, dtype=np.bool_)
60
60
  self.priority_memory = np.zeros(max_size, dtype=np.float32)
61
-
62
- def put(self, state, action, reward, next_state, terminal):
63
- """Store one transition into the replay buffer."""
64
- index = self.collected_size % self.max_size
65
-
66
- self.state_memory[index] = state
67
- self.action_memory[index] = action
68
- self.reward_memory[index] = reward
69
- self.next_state_memory[index] = next_state
70
- self.terminal_memory[index] = terminal
71
- self.priority_memory[index] = self.max_priority
72
-
73
- self.collected_size += 1
74
- self.available_size = min(self.max_size, self.collected_size)
61
+ self.extra_data_memory = {}
75
62
 
76
63
  def add(self, rollout):
77
- """Store transitions collected by a runtime sampler."""
78
- actions = rollout["policy_data"]["action"]
79
- transitions = zip(
80
- rollout["states"],
81
- actions,
82
- rollout["rewards"],
83
- rollout["next_states"],
84
- rollout["terminals"],
85
- )
86
- for transition in transitions:
87
- self.put(*transition)
64
+ """Store a rollout with leading [num_envs, num_steps] dimensions."""
65
+ count = rollout["rewards"].size
66
+ size = min(count, self.max_size)
67
+ offset = count - size
68
+ start = (self.collected_size + offset) % self.max_size
69
+ end = min(start + size, self.max_size)
70
+ first_size = end - start
71
+ remaining_size = size - first_size
72
+
73
+ # Initialize extra data storage.
74
+ extra_data = rollout["extra_data"]
75
+ if self.collected_size == 0:
76
+ for name, values in extra_data.items():
77
+ memory = np.empty((self.max_size, *values.shape[2:]), dtype=values.dtype)
78
+ self.extra_data_memory[name] = memory
79
+
80
+ state_shape = self.state_memory.shape[1:]
81
+ action_shape = self.action_memory.shape[1:]
82
+ states = rollout["states"].reshape(count, *state_shape)[offset:]
83
+ actions = rollout["action"].reshape(count, *action_shape)[offset:]
84
+ rewards = rollout["rewards"].reshape(count)[offset:]
85
+ next_states = rollout["next_states"].reshape(count, *state_shape)[offset:]
86
+ terminals = rollout["terminals"].reshape(count)[offset:]
87
+
88
+ self.state_memory[start:end] = states[:first_size]
89
+ self.action_memory[start:end] = actions[:first_size]
90
+ self.reward_memory[start:end] = rewards[:first_size]
91
+ self.next_state_memory[start:end] = next_states[:first_size]
92
+ self.terminal_memory[start:end] = terminals[:first_size]
93
+ self.priority_memory[start:end] = self.max_priority
94
+
95
+ self.state_memory[:remaining_size] = states[first_size:]
96
+ self.action_memory[:remaining_size] = actions[first_size:]
97
+ self.reward_memory[:remaining_size] = rewards[first_size:]
98
+ self.next_state_memory[:remaining_size] = next_states[first_size:]
99
+ self.terminal_memory[:remaining_size] = terminals[first_size:]
100
+ self.priority_memory[:remaining_size] = self.max_priority
101
+ for name, values in extra_data.items():
102
+ values = values.reshape(count, *values.shape[2:])[offset:]
103
+ memory = self.extra_data_memory[name]
104
+ memory[start:end] = values[:first_size]
105
+ memory[:remaining_size] = values[first_size:]
106
+
107
+ self.collected_size += count
108
+ self.available_size = min(self.max_size, self.collected_size)
88
109
 
89
110
  def get_batch(self, rank, world_size):
90
111
  """Sample a rank-specific mini-batch from the replay buffer."""
@@ -107,7 +128,11 @@ class PrioritizedReplayBuffer:
107
128
  max_weight = (self.available_size * probs.min()) ** (-self.beta)
108
129
  weights /= max_weight
109
130
 
110
- return {
131
+ extra_data = {}
132
+ for name, memory in self.extra_data_memory.items():
133
+ extra_data[name] = memory[indices]
134
+
135
+ batch = {
111
136
  "states": self.state_memory[indices],
112
137
  "actions": self.action_memory[indices],
113
138
  "rewards": self.reward_memory[indices],
@@ -115,7 +140,9 @@ class PrioritizedReplayBuffer:
115
140
  "terminals": self.terminal_memory[indices],
116
141
  "indices": indices,
117
142
  "weights": weights.astype(np.float32),
143
+ **extra_data,
118
144
  }
145
+ return batch
119
146
 
120
147
  def on_learn(self, train_infos):
121
148
  """Update internal state after the learner finishes one training step."""
@@ -149,14 +176,6 @@ class PrioritizedReplayBuffer:
149
176
  """Return True because batches can be sampled before the last update."""
150
177
  return True
151
178
 
152
- def _clear(self):
153
- """Clear all stored transitions in the replay buffer."""
154
- self.available_size = 0
155
- self.collected_size = 0
156
- self.replayed_size = 0
157
- self.beta = self.beta_start
158
- self.max_priority = 1.0
159
-
160
179
  def stats(self):
161
180
  """Return statistics describing the current state of the replay buffer."""
162
181
  return {
@@ -45,32 +45,52 @@ class ReplayBuffer:
45
45
  self.reward_memory = np.zeros(max_size, dtype=np.float32)
46
46
  self.next_state_memory = np.zeros((max_size, *state_dim), dtype=state_dtype)
47
47
  self.terminal_memory = np.zeros(max_size, dtype=np.bool_)
48
-
49
- def put(self, state, action, reward, next_state, terminal):
50
- """Store one transition into the replay buffer."""
51
- index = self.collected_size % self.max_size
52
-
53
- self.state_memory[index] = state
54
- self.action_memory[index] = action
55
- self.reward_memory[index] = reward
56
- self.next_state_memory[index] = next_state
57
- self.terminal_memory[index] = terminal
58
-
59
- self.collected_size += 1
60
- self.available_size = min(self.max_size, self.collected_size)
48
+ self.extra_data_memory = {}
61
49
 
62
50
  def add(self, rollout):
63
- """Store transitions collected by a runtime sampler."""
64
- actions = rollout["policy_data"]["action"]
65
- transitions = zip(
66
- rollout["states"],
67
- actions,
68
- rollout["rewards"],
69
- rollout["next_states"],
70
- rollout["terminals"],
71
- )
72
- for transition in transitions:
73
- self.put(*transition)
51
+ """Store a rollout with leading [num_envs, num_steps] dimensions."""
52
+ count = rollout["rewards"].size
53
+ size = min(count, self.max_size)
54
+ offset = count - size
55
+ start = (self.collected_size + offset) % self.max_size
56
+ end = min(start + size, self.max_size)
57
+ first_size = end - start
58
+ remaining_size = size - first_size
59
+
60
+ # Initialize extra data storage.
61
+ extra_data = rollout["extra_data"]
62
+ if self.collected_size == 0:
63
+ for name, values in extra_data.items():
64
+ memory = np.empty((self.max_size, *values.shape[2:]), dtype=values.dtype)
65
+ self.extra_data_memory[name] = memory
66
+
67
+ state_shape = self.state_memory.shape[1:]
68
+ action_shape = self.action_memory.shape[1:]
69
+ states = rollout["states"].reshape(count, *state_shape)[offset:]
70
+ actions = rollout["action"].reshape(count, *action_shape)[offset:]
71
+ rewards = rollout["rewards"].reshape(count)[offset:]
72
+ next_states = rollout["next_states"].reshape(count, *state_shape)[offset:]
73
+ terminals = rollout["terminals"].reshape(count)[offset:]
74
+
75
+ self.state_memory[start:end] = states[:first_size]
76
+ self.action_memory[start:end] = actions[:first_size]
77
+ self.reward_memory[start:end] = rewards[:first_size]
78
+ self.next_state_memory[start:end] = next_states[:first_size]
79
+ self.terminal_memory[start:end] = terminals[:first_size]
80
+
81
+ self.state_memory[:remaining_size] = states[first_size:]
82
+ self.action_memory[:remaining_size] = actions[first_size:]
83
+ self.reward_memory[:remaining_size] = rewards[first_size:]
84
+ self.next_state_memory[:remaining_size] = next_states[first_size:]
85
+ self.terminal_memory[:remaining_size] = terminals[first_size:]
86
+ for name, values in extra_data.items():
87
+ values = values.reshape(count, *values.shape[2:])[offset:]
88
+ memory = self.extra_data_memory[name]
89
+ memory[start:end] = values[:first_size]
90
+ memory[:remaining_size] = values[first_size:]
91
+
92
+ self.collected_size += count
93
+ self.available_size = min(self.max_size, self.collected_size)
74
94
 
75
95
  def get_batch(self, rank, world_size):
76
96
  """Sample a rank-specific mini-batch from the replay buffer."""
@@ -84,13 +104,19 @@ class ReplayBuffer:
84
104
 
85
105
  indices = np.random.choice(self.available_size, size=size, replace=False)
86
106
 
87
- return {
107
+ extra_data = {}
108
+ for name, memory in self.extra_data_memory.items():
109
+ extra_data[name] = memory[indices]
110
+
111
+ batch = {
88
112
  "states": self.state_memory[indices],
89
113
  "actions": self.action_memory[indices],
90
114
  "rewards": self.reward_memory[indices],
91
115
  "next_states": self.next_state_memory[indices],
92
116
  "terminals": self.terminal_memory[indices],
117
+ **extra_data,
93
118
  }
119
+ return batch
94
120
 
95
121
  def on_learn(self, train_infos):
96
122
  """Update internal state after the learner finishes one training step."""
@@ -106,12 +132,6 @@ class ReplayBuffer:
106
132
  """Return True because batches can be sampled before the last update."""
107
133
  return True
108
134
 
109
- def _clear(self):
110
- """Clear all stored transitions in the replay buffer."""
111
- self.available_size = 0
112
- self.collected_size = 0
113
- self.replayed_size = 0
114
-
115
135
  def stats(self):
116
136
  """Return statistics describing the current state of the replay buffer."""
117
137
  return {