omnirl-framework 0.3.2__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {omnirl_framework-0.3.2/omnirl_framework.egg-info → omnirl_framework-0.3.4}/PKG-INFO +1 -1
  2. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/buffers/prioritized_replay_buffer.py +2 -0
  3. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/buffers/replay_buffer.py +4 -1
  4. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/buffers/rollout_buffer.py +7 -7
  5. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/runner.py +4 -2
  6. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/sampler.py +9 -6
  7. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4/omnirl_framework.egg-info}/PKG-INFO +1 -1
  8. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/pyproject.toml +1 -1
  9. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/tests/test_integration.py +9 -7
  10. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/tests/test_runtime.py +30 -11
  11. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/tests/test_structure.py +6 -0
  12. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/LICENSE +0 -0
  13. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/README.md +0 -0
  14. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/__init__.py +0 -0
  15. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/__init__.py +0 -0
  16. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/a2c/__init__.py +0 -0
  17. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/a2c/learner.py +0 -0
  18. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/a2c/policy.py +0 -0
  19. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/c51_dqn/__init__.py +0 -0
  20. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/c51_dqn/learner.py +0 -0
  21. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/c51_dqn/policy.py +0 -0
  22. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddpg/__init__.py +0 -0
  23. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddpg/learner.py +0 -0
  24. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddpg/policy.py +0 -0
  25. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddqn/__init__.py +0 -0
  26. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddqn/learner.py +0 -0
  27. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ddqn/policy.py +0 -0
  28. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dqn/__init__.py +0 -0
  29. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dqn/learner.py +0 -0
  30. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dqn/policy.py +0 -0
  31. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dueling_dqn/__init__.py +0 -0
  32. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dueling_dqn/learner.py +0 -0
  33. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/dueling_dqn/policy.py +0 -0
  34. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/impala/__init__.py +0 -0
  35. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/impala/learner.py +0 -0
  36. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/impala/policy.py +0 -0
  37. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/per_dqn/__init__.py +0 -0
  38. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/per_dqn/learner.py +0 -0
  39. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/per_dqn/policy.py +0 -0
  40. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ppo/__init__.py +0 -0
  41. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ppo/learner.py +0 -0
  42. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/ppo/policy.py +0 -0
  43. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/qr_dqn/__init__.py +0 -0
  44. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/qr_dqn/learner.py +0 -0
  45. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/qr_dqn/policy.py +0 -0
  46. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/sac/__init__.py +0 -0
  47. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/sac/learner.py +0 -0
  48. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/sac/policy.py +0 -0
  49. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/td3/__init__.py +0 -0
  50. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/td3/learner.py +0 -0
  51. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/td3/policy.py +0 -0
  52. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/tqc/__init__.py +0 -0
  53. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/tqc/learner.py +0 -0
  54. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/algorithms/tqc/policy.py +0 -0
  55. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/buffers/__init__.py +0 -0
  56. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/__init__.py +0 -0
  57. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/__init__.py +0 -0
  58. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/adapter.py +0 -0
  59. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/builder.py +0 -0
  60. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/collection.py +0 -0
  61. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/evaluation.py +0 -0
  62. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/envs/gymnasium/spec.py +0 -0
  63. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/__init__.py +0 -0
  64. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/actors.py +1 -1
  65. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/base/__init__.py +0 -0
  66. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/base/cnn.py +0 -0
  67. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/base/mlp.py +0 -0
  68. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/base/resnet.py +0 -0
  69. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/networks/critics.py +0 -0
  70. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/__init__.py +0 -0
  71. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/evaluator.py +0 -0
  72. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/file_writer.py +0 -0
  73. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/logger.py +0 -0
  74. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/runtime/weight_store.py +0 -0
  75. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/utils/__init__.py +0 -0
  76. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl/utils/seed.py +0 -0
  77. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl_framework.egg-info/SOURCES.txt +0 -0
  78. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl_framework.egg-info/dependency_links.txt +0 -0
  79. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl_framework.egg-info/requires.txt +0 -0
  80. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/omnirl_framework.egg-info/top_level.txt +0 -0
  81. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/requirements/base.txt +0 -0
  82. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/setup.cfg +0 -0
  83. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/tests/test_gymnasium.py +0 -0
  84. {omnirl_framework-0.3.2 → omnirl_framework-0.3.4}/tests/test_networks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: omnirl-framework
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: A distributed reinforcement learning framework.
5
5
  Author-email: Huang Chenbiao <a1057951125@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -18,6 +18,7 @@ class PrioritizedReplayBuffer:
18
18
  batch_size: Mini-batch size sampled from the buffer.
19
19
  warmup_size: Minimum number of transitions required before training.
20
20
  max_size: Maximum number of replay transitions stored in the buffer.
21
+ collection_fragment_length: Number of steps collected per environment.
21
22
  """
22
23
 
23
24
  def __init__(
@@ -32,6 +33,7 @@ class PrioritizedReplayBuffer:
32
33
  batch_size: int = 64,
33
34
  warmup_size: int = 0,
34
35
  max_size: int = 1000000,
36
+ collection_fragment_length: int = 1,
35
37
  ):
36
38
  self.batch_size = batch_size
37
39
  self.warmup_size = warmup_size
@@ -15,6 +15,7 @@ class ReplayBuffer:
15
15
  batch_size: Mini-batch size sampled from the buffer.
16
16
  warmup_size: Minimum number of transitions required before training.
17
17
  max_size: Maximum number of replay transitions stored in the buffer.
18
+ collection_fragment_length: Number of steps collected per environment.
18
19
  """
19
20
 
20
21
  def __init__(
@@ -26,10 +27,12 @@ class ReplayBuffer:
26
27
  batch_size: int = 64,
27
28
  warmup_size: int = 0,
28
29
  max_size: int = 1000000,
30
+ collection_fragment_length: int = 1,
29
31
  ):
30
32
  self.batch_size = batch_size
31
33
  self.warmup_size = warmup_size
32
34
  self.max_size = max_size
35
+ self.rng = np.random.default_rng()
33
36
 
34
37
  self.collected_size = 0
35
38
  self.available_size = 0
@@ -102,7 +105,7 @@ class ReplayBuffer:
102
105
  end = self.batch_size if rank == world_size - 1 else start + batch_per_worker
103
106
  size = end - start
104
107
 
105
- indices = np.random.choice(self.available_size, size=size, replace=False)
108
+ indices = self.rng.choice(self.available_size, size=size, replace=False)
106
109
 
107
110
  extra_data = {}
108
111
  for name, memory in self.extra_data_memory.items():
@@ -12,8 +12,8 @@ class RolloutBuffer:
12
12
  state_dtype: Data type of the environment state space.
13
13
  action_dim: Dimension of the environment action space.
14
14
  is_continuous: Whether the action space is continuous.
15
- unroll_length: Number of transitions contained in each rollout segment.
16
- max_chunks: Maximum number of rollout chunks stored in the buffer.
15
+ batch_size: Transitions per update, rounded down to complete chunks.
16
+ collection_fragment_length: Number of steps collected per environment.
17
17
  track_version: Whether to reject stale rollouts and track learner versions.
18
18
  """
19
19
 
@@ -23,12 +23,12 @@ class RolloutBuffer:
23
23
  state_dtype: np.dtype | type,
24
24
  action_dim: int,
25
25
  is_continuous: bool = False,
26
- unroll_length: int = 32,
27
- max_chunks: int = 64,
26
+ batch_size: int = 2048,
27
+ collection_fragment_length: int = 1,
28
28
  track_version: bool = True,
29
29
  ):
30
- self.unroll_length = unroll_length
31
- self.max_chunks = max_chunks
30
+ self.unroll_length = collection_fragment_length
31
+ self.max_chunks = batch_size // self.unroll_length
32
32
  self.track_version = track_version
33
33
 
34
34
  self.collected_chunks = 0
@@ -38,7 +38,7 @@ class RolloutBuffer:
38
38
  self.version = -1
39
39
 
40
40
  state_dim = (state_dim,) if isinstance(state_dim, int) else tuple(state_dim)
41
- chunk_shape = (max_chunks, unroll_length)
41
+ chunk_shape = (self.max_chunks, self.unroll_length)
42
42
 
43
43
  self.state_memory = np.zeros((*chunk_shape, *state_dim), dtype=state_dtype)
44
44
  if is_continuous:
@@ -28,6 +28,7 @@ class Runner:
28
28
  num_learners: Number of learner actors.
29
29
  num_samplers: Number of sampler actors.
30
30
  num_envs_per_sampler: Number of environments sharing each sampler policy.
31
+ collection_fragment_length: Steps collected per environment per transfer.
31
32
  num_evaluators: Number of evaluator actors.
32
33
  num_episodes: Number of evaluation episodes per evaluator.
33
34
  max_updates: Maximum number of learner updates.
@@ -53,6 +54,7 @@ class Runner:
53
54
  num_learners: int = 1,
54
55
  num_samplers: int = 1,
55
56
  num_envs_per_sampler: int = 1,
57
+ collection_fragment_length: int = 1,
56
58
  num_evaluators: int = 1,
57
59
  num_episodes: int = 10,
58
60
  max_updates: int = 10000000,
@@ -95,9 +97,9 @@ class Runner:
95
97
  memory_kwargs = dict(agent_config.get("memory") or {})
96
98
  policy_kwargs = dict(agent_config.get("policy") or {})
97
99
  network_kwargs = dict(agent_config.get("network") or {})
98
- unroll_length = memory_kwargs.get("unroll_length", 1)
99
100
 
100
101
  # Initialize memory
102
+ memory_kwargs["collection_fragment_length"] = collection_fragment_length
101
103
  self.memory = agent.Memory.remote(**state_spec, **memory_kwargs)
102
104
  supports_prefetch = ray.get(self.memory.supports_prefetch.remote())
103
105
 
@@ -140,8 +142,8 @@ class Runner:
140
142
  action_spec=action_spec,
141
143
  network_kwargs=network_kwargs,
142
144
  policy_kwargs=policy_kwargs,
143
- unroll_length=unroll_length,
144
145
  num_envs_per_sampler=num_envs_per_sampler,
146
+ collection_fragment_length=collection_fragment_length,
145
147
  seed=seed,
146
148
  )
147
149
  )
@@ -16,9 +16,9 @@ class Sampler:
16
16
  action_spec: Environment arguments for policy initialization.
17
17
  network_kwargs: Keyword arguments used to build the policy network.
18
18
  policy_kwargs: Keyword arguments used to build the policy.
19
- unroll_length: Number of transitions collected per environment per chunk.
20
- seed: Random seed used to ensure consistent results across runs.
21
19
  num_envs_per_sampler: Number of environments sharing one inference batch.
20
+ collection_fragment_length: Steps collected per environment per transfer.
21
+ seed: Random seed used to ensure consistent results across runs.
22
22
  """
23
23
 
24
24
  def __init__(
@@ -29,8 +29,8 @@ class Sampler:
29
29
  action_spec,
30
30
  network_kwargs: dict | None = None,
31
31
  policy_kwargs: dict | None = None,
32
- unroll_length: int = 1,
33
32
  num_envs_per_sampler: int = 1,
33
+ collection_fragment_length: int = 1,
34
34
  seed: int | None = None,
35
35
  ):
36
36
  # Limit PyTorch CPU threads
@@ -52,7 +52,7 @@ class Sampler:
52
52
  seed=seed,
53
53
  num_envs=num_envs_per_sampler,
54
54
  )
55
- self.unroll_length = unroll_length
55
+ self.fragment_length = collection_fragment_length
56
56
  self.version = -1
57
57
 
58
58
  def _sync_weights(self, weight_store):
@@ -65,8 +65,11 @@ class Sampler:
65
65
 
66
66
  def rollout(self, memory, weight_store):
67
67
  """Continuously collect environment data and send it to the buffer."""
68
+ pending_add = None
68
69
  while True:
69
70
  self._sync_weights(weight_store)
70
- rollout = self.collection.collect(self.policy, self.unroll_length)
71
+ rollout = self.collection.collect(self.policy, self.fragment_length)
71
72
  rollout["version"] = self.version
72
- memory.add.remote(rollout)
73
+ if pending_add is not None:
74
+ ray.get(pending_add)
75
+ pending_add = memory.add.remote(rollout)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: omnirl-framework
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: A distributed reinforcement learning framework.
5
5
  Author-email: Huang Chenbiao <a1057951125@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "omnirl-framework"
7
- version = "0.3.2"
7
+ version = "0.3.4"
8
8
  description = "A distributed reinforcement learning framework."
9
9
  readme = "README.md"
10
10
  authors = [
@@ -29,22 +29,22 @@ class IntegrationTest(unittest.TestCase):
29
29
  },
30
30
  }
31
31
 
32
- for num_envs in (1, 4):
33
- with self.subTest(num_envs=num_envs):
34
- self.run_one_update(dqn, agent_config, num_envs)
32
+ for num_envs, fragment_length in ((1, 1), (4, 3)):
33
+ with self.subTest(num_envs=num_envs, fragment_length=fragment_length):
34
+ self.run_one_update(dqn, agent_config, num_envs, fragment_length)
35
35
 
36
36
  def test_ppo_runner_one_update(self):
37
37
  agent_config = {
38
38
  "learner": {"k_epochs": 1, "num_minibatches": 1},
39
39
  "memory": {
40
- "unroll_length": 2,
41
- "max_chunks": 3,
40
+ "batch_size": 6,
42
41
  },
43
42
  "network": {"layer_type": "MLP", "mlp_hidden_dims": [8, 8]},
44
43
  }
45
- self.run_one_update(ppo, agent_config, num_envs=4)
44
+ trained_samples = self.run_one_update(ppo, agent_config, num_envs=4, fragment_length=3)
45
+ self.assertEqual(trained_samples, 6)
46
46
 
47
- def run_one_update(self, agent, agent_config, num_envs):
47
+ def run_one_update(self, agent, agent_config, num_envs, fragment_length=1):
48
48
  with tempfile.TemporaryDirectory() as ckpt_dir:
49
49
  runner_kwargs = {
50
50
  "agent": agent,
@@ -55,6 +55,7 @@ class IntegrationTest(unittest.TestCase):
55
55
  "num_learners": 1,
56
56
  "num_samplers": 1,
57
57
  "num_envs_per_sampler": num_envs,
58
+ "collection_fragment_length": fragment_length,
58
59
  "num_evaluators": 1,
59
60
  "num_episodes": 1,
60
61
  "max_updates": 1,
@@ -71,6 +72,7 @@ class IntegrationTest(unittest.TestCase):
71
72
  runner.fit()
72
73
  self.assertEqual(runner.version, 1)
73
74
  self.assertGreater(runner.trained_samples, 0)
75
+ return runner.trained_samples
74
76
  finally:
75
77
  ray.shutdown()
76
78
 
@@ -93,6 +93,7 @@ class RuntimeTest(unittest.TestCase):
93
93
  "batch_size": 2,
94
94
  "warmup_size": 2,
95
95
  "max_size": 8,
96
+ "collection_fragment_length": 3,
96
97
  }
97
98
  memory = ReplayBuffer.remote(**memory_kwargs)
98
99
 
@@ -129,6 +130,7 @@ class RuntimeTest(unittest.TestCase):
129
130
  "batch_size": 2,
130
131
  "warmup_size": 2,
131
132
  "max_size": 8,
133
+ "collection_fragment_length": 3,
132
134
  }
133
135
  memory = PrioritizedReplayBuffer.remote(**memory_kwargs)
134
136
 
@@ -238,30 +240,47 @@ class RuntimeTest(unittest.TestCase):
238
240
  "state_dim": 4,
239
241
  "state_dtype": np.float32,
240
242
  "action_dim": 2,
241
- "unroll_length": 2,
242
- "max_chunks": 1,
243
+ "batch_size": 5,
244
+ "collection_fragment_length": 2,
243
245
  }
244
246
  memory = RolloutBuffer.remote(**memory_kwargs)
247
+ self.assertFalse(ray.get(memory.ready.remote()))
248
+ self.assertIsNone(ray.get(memory.get_batch.remote(0, 1)))
249
+ self.assertEqual(
250
+ ray.get(memory.stats.remote()),
251
+ {"available_data": 0, "collected_data": 0},
252
+ )
245
253
 
246
254
  rollout = {
247
255
  "version": 0,
248
- "states": np.zeros((1, 2, 4), dtype=np.float32),
249
- "next_states": np.ones((1, 2, 4), dtype=np.float32),
250
- "rewards": np.zeros((1, 2), dtype=np.float32),
251
- "terminals": np.zeros((1, 2), dtype=np.bool_),
252
- "dones": np.zeros((1, 2), dtype=np.bool_),
253
- "action": np.zeros((1, 2), dtype=np.int64),
256
+ "states": np.arange(24, dtype=np.float32).reshape(3, 2, 4),
257
+ "next_states": np.ones((3, 2, 4), dtype=np.float32),
258
+ "rewards": np.zeros((3, 2), dtype=np.float32),
259
+ "terminals": np.zeros((3, 2), dtype=np.bool_),
260
+ "dones": np.zeros((3, 2), dtype=np.bool_),
261
+ "action": np.zeros((3, 2), dtype=np.int64),
254
262
  "extra_data": {
255
- "log_probs": np.zeros((1, 2), dtype=np.float32),
263
+ "log_probs": np.zeros((3, 2), dtype=np.float32),
256
264
  },
257
265
  }
258
266
  ray.get(memory.add.remote(rollout))
259
267
 
260
268
  self.assertTrue(ray.get(memory.ready.remote()))
261
269
  batch = ray.get(memory.get_batch.remote(0, 1))
262
- self.assertEqual(batch["states"].shape, (1, 2, 4))
263
- self.assertEqual(batch["log_probs"].shape, (1, 2))
270
+ self.assertEqual(batch["states"].shape, (2, 2, 4))
271
+ self.assertEqual(batch["log_probs"].shape, (2, 2))
264
272
  self.assertNotIn("extra_data", batch)
273
+ np.testing.assert_array_equal(batch["states"], rollout["states"][:2])
274
+ self.assertEqual(
275
+ ray.get(memory.stats.remote()),
276
+ {"available_data": 4, "collected_data": 4},
277
+ )
278
+
279
+ ray.get(memory.on_learn.remote([{"version": 1}]))
280
+ self.assertFalse(ray.get(memory.ready.remote()))
281
+ rollout["version"] = 1
282
+ ray.get(memory.add.remote(rollout))
283
+ self.assertTrue(ray.get(memory.ready.remote()))
265
284
 
266
285
  def test_representative_learners(self):
267
286
  from omnirl.algorithms.ddpg import Learner as DDPGLearner
@@ -67,6 +67,10 @@ class StructureTest(unittest.TestCase):
67
67
  (example_dir / "test.yaml").read_text(encoding="utf-8")
68
68
  )
69
69
  self.assertEqual(agent["agent"]["module"], "omnirl.algorithms.ppo")
70
+ self.assertGreaterEqual(
71
+ agent["modules"]["memory"]["batch_size"],
72
+ train["runner"]["collection_fragment_length"],
73
+ )
70
74
  for component in ("actor", "critic"):
71
75
  self.assertEqual(
72
76
  agent["modules"]["network"][component]["layer_type"], layer_type
@@ -153,6 +157,7 @@ class StructureTest(unittest.TestCase):
153
157
  runner_config = yaml.safe_load(file)
154
158
  with (example_dir / "ppo.yaml").open("r", encoding="utf-8") as file:
155
159
  agent_config = yaml.safe_load(file)
160
+ runner_config["runner"]["collection_fragment_length"] = 8
156
161
 
157
162
  agent = object()
158
163
  environment = object()
@@ -166,6 +171,7 @@ class StructureTest(unittest.TestCase):
166
171
  expected_env_config = {"id": "CartPole-v1", "render_mode": None}
167
172
  self.assertEqual(captured_kwargs["env_config"], expected_env_config)
168
173
  self.assertEqual(captured_kwargs["agent_config"], agent_config["modules"])
174
+ self.assertEqual(captured_kwargs["collection_fragment_length"], 8)
169
175
  self.assertEqual(captured_kwargs["learner_seed"], 0)
170
176
  self.assertEqual(captured_kwargs["sampler_seed"], 0)
171
177
  self.assertEqual(captured_kwargs["evaluator_seed"], 0)
@@ -11,8 +11,8 @@ class StochasticActorNetwork(nn.Module):
11
11
  Args:
12
12
  state_dim: Dimension of the environment state space.
13
13
  action_dim: Dimension of the environment action space.
14
- is_continuous: Whether the action space is continuous.
15
14
  layer_type: Type of feature network to use.
15
+ is_continuous: Whether the action space is continuous.
16
16
  log_std_min: Minimum value for log standard deviation.
17
17
  log_std_max: Maximum value for log standard deviation.
18
18
  network_kwargs: Keyword arguments used to build the feature network.