omnirl-framework 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omnirl_framework-0.3.2/omnirl_framework.egg-info → omnirl_framework-0.3.3}/PKG-INFO +1 -1
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/buffers/prioritized_replay_buffer.py +2 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/buffers/replay_buffer.py +2 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/buffers/rollout_buffer.py +7 -7
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/runner.py +4 -2
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/sampler.py +9 -6
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3/omnirl_framework.egg-info}/PKG-INFO +1 -1
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/pyproject.toml +1 -1
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/tests/test_integration.py +9 -7
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/tests/test_runtime.py +30 -11
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/tests/test_structure.py +6 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/LICENSE +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/README.md +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/a2c/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/a2c/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/a2c/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/c51_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/c51_dqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/c51_dqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddpg/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddpg/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddpg/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ddqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dueling_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dueling_dqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/dueling_dqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/impala/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/impala/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/impala/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/per_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/per_dqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/per_dqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ppo/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ppo/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/ppo/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/qr_dqn/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/qr_dqn/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/qr_dqn/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/sac/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/sac/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/sac/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/td3/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/td3/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/td3/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/tqc/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/tqc/learner.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/algorithms/tqc/policy.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/buffers/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/adapter.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/builder.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/collection.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/evaluation.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/envs/gymnasium/spec.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/actors.py +1 -1
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/base/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/base/cnn.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/base/mlp.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/base/resnet.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/networks/critics.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/evaluator.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/file_writer.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/logger.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/runtime/weight_store.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/utils/__init__.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/utils/seed.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl_framework.egg-info/SOURCES.txt +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl_framework.egg-info/dependency_links.txt +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl_framework.egg-info/requires.txt +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl_framework.egg-info/top_level.txt +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/requirements/base.txt +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/setup.cfg +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/tests/test_gymnasium.py +0 -0
- {omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/tests/test_networks.py +0 -0
{omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl/buffers/prioritized_replay_buffer.py
RENAMED
|
@@ -18,6 +18,7 @@ class PrioritizedReplayBuffer:
|
|
|
18
18
|
batch_size: Mini-batch size sampled from the buffer.
|
|
19
19
|
warmup_size: Minimum number of transitions required before training.
|
|
20
20
|
max_size: Maximum number of replay transitions stored in the buffer.
|
|
21
|
+
collection_fragment_length: Number of steps collected per environment.
|
|
21
22
|
"""
|
|
22
23
|
|
|
23
24
|
def __init__(
|
|
@@ -32,6 +33,7 @@ class PrioritizedReplayBuffer:
|
|
|
32
33
|
batch_size: int = 64,
|
|
33
34
|
warmup_size: int = 0,
|
|
34
35
|
max_size: int = 1000000,
|
|
36
|
+
collection_fragment_length: int = 1,
|
|
35
37
|
):
|
|
36
38
|
self.batch_size = batch_size
|
|
37
39
|
self.warmup_size = warmup_size
|
|
@@ -15,6 +15,7 @@ class ReplayBuffer:
|
|
|
15
15
|
batch_size: Mini-batch size sampled from the buffer.
|
|
16
16
|
warmup_size: Minimum number of transitions required before training.
|
|
17
17
|
max_size: Maximum number of replay transitions stored in the buffer.
|
|
18
|
+
collection_fragment_length: Number of steps collected per environment.
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
21
|
def __init__(
|
|
@@ -26,6 +27,7 @@ class ReplayBuffer:
|
|
|
26
27
|
batch_size: int = 64,
|
|
27
28
|
warmup_size: int = 0,
|
|
28
29
|
max_size: int = 1000000,
|
|
30
|
+
collection_fragment_length: int = 1,
|
|
29
31
|
):
|
|
30
32
|
self.batch_size = batch_size
|
|
31
33
|
self.warmup_size = warmup_size
|
|
@@ -12,8 +12,8 @@ class RolloutBuffer:
|
|
|
12
12
|
state_dtype: Data type of the environment state space.
|
|
13
13
|
action_dim: Dimension of the environment action space.
|
|
14
14
|
is_continuous: Whether the action space is continuous.
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
batch_size: Transitions per update, rounded down to complete chunks.
|
|
16
|
+
collection_fragment_length: Number of steps collected per environment.
|
|
17
17
|
track_version: Whether to reject stale rollouts and track learner versions.
|
|
18
18
|
"""
|
|
19
19
|
|
|
@@ -23,12 +23,12 @@ class RolloutBuffer:
|
|
|
23
23
|
state_dtype: np.dtype | type,
|
|
24
24
|
action_dim: int,
|
|
25
25
|
is_continuous: bool = False,
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
batch_size: int = 2048,
|
|
27
|
+
collection_fragment_length: int = 1,
|
|
28
28
|
track_version: bool = True,
|
|
29
29
|
):
|
|
30
|
-
self.unroll_length =
|
|
31
|
-
self.max_chunks =
|
|
30
|
+
self.unroll_length = collection_fragment_length
|
|
31
|
+
self.max_chunks = batch_size // self.unroll_length
|
|
32
32
|
self.track_version = track_version
|
|
33
33
|
|
|
34
34
|
self.collected_chunks = 0
|
|
@@ -38,7 +38,7 @@ class RolloutBuffer:
|
|
|
38
38
|
self.version = -1
|
|
39
39
|
|
|
40
40
|
state_dim = (state_dim,) if isinstance(state_dim, int) else tuple(state_dim)
|
|
41
|
-
chunk_shape = (max_chunks, unroll_length)
|
|
41
|
+
chunk_shape = (self.max_chunks, self.unroll_length)
|
|
42
42
|
|
|
43
43
|
self.state_memory = np.zeros((*chunk_shape, *state_dim), dtype=state_dtype)
|
|
44
44
|
if is_continuous:
|
|
@@ -28,6 +28,7 @@ class Runner:
|
|
|
28
28
|
num_learners: Number of learner actors.
|
|
29
29
|
num_samplers: Number of sampler actors.
|
|
30
30
|
num_envs_per_sampler: Number of environments sharing each sampler policy.
|
|
31
|
+
collection_fragment_length: Steps collected per environment per transfer.
|
|
31
32
|
num_evaluators: Number of evaluator actors.
|
|
32
33
|
num_episodes: Number of evaluation episodes per evaluator.
|
|
33
34
|
max_updates: Maximum number of learner updates.
|
|
@@ -53,6 +54,7 @@ class Runner:
|
|
|
53
54
|
num_learners: int = 1,
|
|
54
55
|
num_samplers: int = 1,
|
|
55
56
|
num_envs_per_sampler: int = 1,
|
|
57
|
+
collection_fragment_length: int = 1,
|
|
56
58
|
num_evaluators: int = 1,
|
|
57
59
|
num_episodes: int = 10,
|
|
58
60
|
max_updates: int = 10000000,
|
|
@@ -95,9 +97,9 @@ class Runner:
|
|
|
95
97
|
memory_kwargs = dict(agent_config.get("memory") or {})
|
|
96
98
|
policy_kwargs = dict(agent_config.get("policy") or {})
|
|
97
99
|
network_kwargs = dict(agent_config.get("network") or {})
|
|
98
|
-
unroll_length = memory_kwargs.get("unroll_length", 1)
|
|
99
100
|
|
|
100
101
|
# Initialize memory
|
|
102
|
+
memory_kwargs["collection_fragment_length"] = collection_fragment_length
|
|
101
103
|
self.memory = agent.Memory.remote(**state_spec, **memory_kwargs)
|
|
102
104
|
supports_prefetch = ray.get(self.memory.supports_prefetch.remote())
|
|
103
105
|
|
|
@@ -140,8 +142,8 @@ class Runner:
|
|
|
140
142
|
action_spec=action_spec,
|
|
141
143
|
network_kwargs=network_kwargs,
|
|
142
144
|
policy_kwargs=policy_kwargs,
|
|
143
|
-
unroll_length=unroll_length,
|
|
144
145
|
num_envs_per_sampler=num_envs_per_sampler,
|
|
146
|
+
collection_fragment_length=collection_fragment_length,
|
|
145
147
|
seed=seed,
|
|
146
148
|
)
|
|
147
149
|
)
|
|
@@ -16,9 +16,9 @@ class Sampler:
|
|
|
16
16
|
action_spec: Environment arguments for policy initialization.
|
|
17
17
|
network_kwargs: Keyword arguments used to build the policy network.
|
|
18
18
|
policy_kwargs: Keyword arguments used to build the policy.
|
|
19
|
-
unroll_length: Number of transitions collected per environment per chunk.
|
|
20
|
-
seed: Random seed used to ensure consistent results across runs.
|
|
21
19
|
num_envs_per_sampler: Number of environments sharing one inference batch.
|
|
20
|
+
collection_fragment_length: Steps collected per environment per transfer.
|
|
21
|
+
seed: Random seed used to ensure consistent results across runs.
|
|
22
22
|
"""
|
|
23
23
|
|
|
24
24
|
def __init__(
|
|
@@ -29,8 +29,8 @@ class Sampler:
|
|
|
29
29
|
action_spec,
|
|
30
30
|
network_kwargs: dict | None = None,
|
|
31
31
|
policy_kwargs: dict | None = None,
|
|
32
|
-
unroll_length: int = 1,
|
|
33
32
|
num_envs_per_sampler: int = 1,
|
|
33
|
+
collection_fragment_length: int = 1,
|
|
34
34
|
seed: int | None = None,
|
|
35
35
|
):
|
|
36
36
|
# Limit PyTorch CPU threads
|
|
@@ -52,7 +52,7 @@ class Sampler:
|
|
|
52
52
|
seed=seed,
|
|
53
53
|
num_envs=num_envs_per_sampler,
|
|
54
54
|
)
|
|
55
|
-
self.
|
|
55
|
+
self.fragment_length = collection_fragment_length
|
|
56
56
|
self.version = -1
|
|
57
57
|
|
|
58
58
|
def _sync_weights(self, weight_store):
|
|
@@ -65,8 +65,11 @@ class Sampler:
|
|
|
65
65
|
|
|
66
66
|
def rollout(self, memory, weight_store):
|
|
67
67
|
"""Continuously collect environment data and send it to the buffer."""
|
|
68
|
+
pending_add = None
|
|
68
69
|
while True:
|
|
69
70
|
self._sync_weights(weight_store)
|
|
70
|
-
rollout = self.collection.collect(self.policy, self.
|
|
71
|
+
rollout = self.collection.collect(self.policy, self.fragment_length)
|
|
71
72
|
rollout["version"] = self.version
|
|
72
|
-
|
|
73
|
+
if pending_add is not None:
|
|
74
|
+
ray.get(pending_add)
|
|
75
|
+
pending_add = memory.add.remote(rollout)
|
|
@@ -29,22 +29,22 @@ class IntegrationTest(unittest.TestCase):
|
|
|
29
29
|
},
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
for num_envs in (1, 4):
|
|
33
|
-
with self.subTest(num_envs=num_envs):
|
|
34
|
-
self.run_one_update(dqn, agent_config, num_envs)
|
|
32
|
+
for num_envs, fragment_length in ((1, 1), (4, 3)):
|
|
33
|
+
with self.subTest(num_envs=num_envs, fragment_length=fragment_length):
|
|
34
|
+
self.run_one_update(dqn, agent_config, num_envs, fragment_length)
|
|
35
35
|
|
|
36
36
|
def test_ppo_runner_one_update(self):
|
|
37
37
|
agent_config = {
|
|
38
38
|
"learner": {"k_epochs": 1, "num_minibatches": 1},
|
|
39
39
|
"memory": {
|
|
40
|
-
"
|
|
41
|
-
"max_chunks": 3,
|
|
40
|
+
"batch_size": 6,
|
|
42
41
|
},
|
|
43
42
|
"network": {"layer_type": "MLP", "mlp_hidden_dims": [8, 8]},
|
|
44
43
|
}
|
|
45
|
-
self.run_one_update(ppo, agent_config, num_envs=4)
|
|
44
|
+
trained_samples = self.run_one_update(ppo, agent_config, num_envs=4, fragment_length=3)
|
|
45
|
+
self.assertEqual(trained_samples, 6)
|
|
46
46
|
|
|
47
|
-
def run_one_update(self, agent, agent_config, num_envs):
|
|
47
|
+
def run_one_update(self, agent, agent_config, num_envs, fragment_length=1):
|
|
48
48
|
with tempfile.TemporaryDirectory() as ckpt_dir:
|
|
49
49
|
runner_kwargs = {
|
|
50
50
|
"agent": agent,
|
|
@@ -55,6 +55,7 @@ class IntegrationTest(unittest.TestCase):
|
|
|
55
55
|
"num_learners": 1,
|
|
56
56
|
"num_samplers": 1,
|
|
57
57
|
"num_envs_per_sampler": num_envs,
|
|
58
|
+
"collection_fragment_length": fragment_length,
|
|
58
59
|
"num_evaluators": 1,
|
|
59
60
|
"num_episodes": 1,
|
|
60
61
|
"max_updates": 1,
|
|
@@ -71,6 +72,7 @@ class IntegrationTest(unittest.TestCase):
|
|
|
71
72
|
runner.fit()
|
|
72
73
|
self.assertEqual(runner.version, 1)
|
|
73
74
|
self.assertGreater(runner.trained_samples, 0)
|
|
75
|
+
return runner.trained_samples
|
|
74
76
|
finally:
|
|
75
77
|
ray.shutdown()
|
|
76
78
|
|
|
@@ -93,6 +93,7 @@ class RuntimeTest(unittest.TestCase):
|
|
|
93
93
|
"batch_size": 2,
|
|
94
94
|
"warmup_size": 2,
|
|
95
95
|
"max_size": 8,
|
|
96
|
+
"collection_fragment_length": 3,
|
|
96
97
|
}
|
|
97
98
|
memory = ReplayBuffer.remote(**memory_kwargs)
|
|
98
99
|
|
|
@@ -129,6 +130,7 @@ class RuntimeTest(unittest.TestCase):
|
|
|
129
130
|
"batch_size": 2,
|
|
130
131
|
"warmup_size": 2,
|
|
131
132
|
"max_size": 8,
|
|
133
|
+
"collection_fragment_length": 3,
|
|
132
134
|
}
|
|
133
135
|
memory = PrioritizedReplayBuffer.remote(**memory_kwargs)
|
|
134
136
|
|
|
@@ -238,30 +240,47 @@ class RuntimeTest(unittest.TestCase):
|
|
|
238
240
|
"state_dim": 4,
|
|
239
241
|
"state_dtype": np.float32,
|
|
240
242
|
"action_dim": 2,
|
|
241
|
-
"
|
|
242
|
-
"
|
|
243
|
+
"batch_size": 5,
|
|
244
|
+
"collection_fragment_length": 2,
|
|
243
245
|
}
|
|
244
246
|
memory = RolloutBuffer.remote(**memory_kwargs)
|
|
247
|
+
self.assertFalse(ray.get(memory.ready.remote()))
|
|
248
|
+
self.assertIsNone(ray.get(memory.get_batch.remote(0, 1)))
|
|
249
|
+
self.assertEqual(
|
|
250
|
+
ray.get(memory.stats.remote()),
|
|
251
|
+
{"available_data": 0, "collected_data": 0},
|
|
252
|
+
)
|
|
245
253
|
|
|
246
254
|
rollout = {
|
|
247
255
|
"version": 0,
|
|
248
|
-
"states": np.
|
|
249
|
-
"next_states": np.ones((
|
|
250
|
-
"rewards": np.zeros((
|
|
251
|
-
"terminals": np.zeros((
|
|
252
|
-
"dones": np.zeros((
|
|
253
|
-
"action": np.zeros((
|
|
256
|
+
"states": np.arange(24, dtype=np.float32).reshape(3, 2, 4),
|
|
257
|
+
"next_states": np.ones((3, 2, 4), dtype=np.float32),
|
|
258
|
+
"rewards": np.zeros((3, 2), dtype=np.float32),
|
|
259
|
+
"terminals": np.zeros((3, 2), dtype=np.bool_),
|
|
260
|
+
"dones": np.zeros((3, 2), dtype=np.bool_),
|
|
261
|
+
"action": np.zeros((3, 2), dtype=np.int64),
|
|
254
262
|
"extra_data": {
|
|
255
|
-
"log_probs": np.zeros((
|
|
263
|
+
"log_probs": np.zeros((3, 2), dtype=np.float32),
|
|
256
264
|
},
|
|
257
265
|
}
|
|
258
266
|
ray.get(memory.add.remote(rollout))
|
|
259
267
|
|
|
260
268
|
self.assertTrue(ray.get(memory.ready.remote()))
|
|
261
269
|
batch = ray.get(memory.get_batch.remote(0, 1))
|
|
262
|
-
self.assertEqual(batch["states"].shape, (
|
|
263
|
-
self.assertEqual(batch["log_probs"].shape, (
|
|
270
|
+
self.assertEqual(batch["states"].shape, (2, 2, 4))
|
|
271
|
+
self.assertEqual(batch["log_probs"].shape, (2, 2))
|
|
264
272
|
self.assertNotIn("extra_data", batch)
|
|
273
|
+
np.testing.assert_array_equal(batch["states"], rollout["states"][:2])
|
|
274
|
+
self.assertEqual(
|
|
275
|
+
ray.get(memory.stats.remote()),
|
|
276
|
+
{"available_data": 4, "collected_data": 4},
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
ray.get(memory.on_learn.remote([{"version": 1}]))
|
|
280
|
+
self.assertFalse(ray.get(memory.ready.remote()))
|
|
281
|
+
rollout["version"] = 1
|
|
282
|
+
ray.get(memory.add.remote(rollout))
|
|
283
|
+
self.assertTrue(ray.get(memory.ready.remote()))
|
|
265
284
|
|
|
266
285
|
def test_representative_learners(self):
|
|
267
286
|
from omnirl.algorithms.ddpg import Learner as DDPGLearner
|
|
@@ -67,6 +67,10 @@ class StructureTest(unittest.TestCase):
|
|
|
67
67
|
(example_dir / "test.yaml").read_text(encoding="utf-8")
|
|
68
68
|
)
|
|
69
69
|
self.assertEqual(agent["agent"]["module"], "omnirl.algorithms.ppo")
|
|
70
|
+
self.assertGreaterEqual(
|
|
71
|
+
agent["modules"]["memory"]["batch_size"],
|
|
72
|
+
train["runner"]["collection_fragment_length"],
|
|
73
|
+
)
|
|
70
74
|
for component in ("actor", "critic"):
|
|
71
75
|
self.assertEqual(
|
|
72
76
|
agent["modules"]["network"][component]["layer_type"], layer_type
|
|
@@ -153,6 +157,7 @@ class StructureTest(unittest.TestCase):
|
|
|
153
157
|
runner_config = yaml.safe_load(file)
|
|
154
158
|
with (example_dir / "ppo.yaml").open("r", encoding="utf-8") as file:
|
|
155
159
|
agent_config = yaml.safe_load(file)
|
|
160
|
+
runner_config["runner"]["collection_fragment_length"] = 8
|
|
156
161
|
|
|
157
162
|
agent = object()
|
|
158
163
|
environment = object()
|
|
@@ -166,6 +171,7 @@ class StructureTest(unittest.TestCase):
|
|
|
166
171
|
expected_env_config = {"id": "CartPole-v1", "render_mode": None}
|
|
167
172
|
self.assertEqual(captured_kwargs["env_config"], expected_env_config)
|
|
168
173
|
self.assertEqual(captured_kwargs["agent_config"], agent_config["modules"])
|
|
174
|
+
self.assertEqual(captured_kwargs["collection_fragment_length"], 8)
|
|
169
175
|
self.assertEqual(captured_kwargs["learner_seed"], 0)
|
|
170
176
|
self.assertEqual(captured_kwargs["sampler_seed"], 0)
|
|
171
177
|
self.assertEqual(captured_kwargs["evaluator_seed"], 0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -11,8 +11,8 @@ class StochasticActorNetwork(nn.Module):
|
|
|
11
11
|
Args:
|
|
12
12
|
state_dim: Dimension of the environment state space.
|
|
13
13
|
action_dim: Dimension of the environment action space.
|
|
14
|
-
is_continuous: Whether the action space is continuous.
|
|
15
14
|
layer_type: Type of feature network to use.
|
|
15
|
+
is_continuous: Whether the action space is continuous.
|
|
16
16
|
log_std_min: Minimum value for log standard deviation.
|
|
17
17
|
log_std_max: Maximum value for log standard deviation.
|
|
18
18
|
network_kwargs: Keyword arguments used to build the feature network.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{omnirl_framework-0.3.2 → omnirl_framework-0.3.3}/omnirl_framework.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|