loco-mujoco 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. loco_mujoco/__init__.py +50 -0
  2. loco_mujoco/algorithms/__init__.py +7 -0
  3. loco_mujoco/algorithms/amp_jax.py +30 -0
  4. loco_mujoco/algorithms/common/__init__.py +2 -0
  5. loco_mujoco/algorithms/common/base_algorithm.py +159 -0
  6. loco_mujoco/algorithms/common/dataclasses.py +381 -0
  7. loco_mujoco/algorithms/common/networks.py +125 -0
  8. loco_mujoco/algorithms/common/policies.py +81 -0
  9. loco_mujoco/algorithms/experimental/__init__.py +7 -0
  10. loco_mujoco/algorithms/experimental/bptt_ppo_jax.py +818 -0
  11. loco_mujoco/algorithms/experimental/history_ppo_jax.py +732 -0
  12. loco_mujoco/algorithms/experimental/offpolicy_base.py +781 -0
  13. loco_mujoco/algorithms/experimental/s2pg_ppo_jax.py +800 -0
  14. loco_mujoco/algorithms/experimental/sac_jax.py +508 -0
  15. loco_mujoco/algorithms/experimental/td3_jax.py +1008 -0
  16. loco_mujoco/algorithms/experimental/vanilla_dagger_jax.py +889 -0
  17. loco_mujoco/algorithms/gail_jax.py +728 -0
  18. loco_mujoco/algorithms/ppo_jax.py +759 -0
  19. loco_mujoco/algorithms/supermimic_jax.py +1767 -0
  20. loco_mujoco/core/__init__.py +10 -0
  21. loco_mujoco/core/control_functions/__init__.py +7 -0
  22. loco_mujoco/core/control_functions/base.py +169 -0
  23. loco_mujoco/core/control_functions/default.py +77 -0
  24. loco_mujoco/core/control_functions/pd.py +191 -0
  25. loco_mujoco/core/domain_randomizer/__init__.py +7 -0
  26. loco_mujoco/core/domain_randomizer/base.py +205 -0
  27. loco_mujoco/core/domain_randomizer/default.py +842 -0
  28. loco_mujoco/core/domain_randomizer/no_randomization.py +108 -0
  29. loco_mujoco/core/initial_state_handler/__init__.py +7 -0
  30. loco_mujoco/core/initial_state_handler/base.py +84 -0
  31. loco_mujoco/core/initial_state_handler/default.py +110 -0
  32. loco_mujoco/core/initial_state_handler/traj_init_state.py +123 -0
  33. loco_mujoco/core/mujoco_base.py +1607 -0
  34. loco_mujoco/core/mujoco_mjx.py +835 -0
  35. loco_mujoco/core/observations/__init__.py +13 -0
  36. loco_mujoco/core/observations/base.py +1190 -0
  37. loco_mujoco/core/observations/goals.py +1217 -0
  38. loco_mujoco/core/observations/visualizer.py +213 -0
  39. loco_mujoco/core/reward/__init__.py +13 -0
  40. loco_mujoco/core/reward/base.py +126 -0
  41. loco_mujoco/core/reward/default.py +519 -0
  42. loco_mujoco/core/reward/trajectory_based.py +648 -0
  43. loco_mujoco/core/reward/utils.py +40 -0
  44. loco_mujoco/core/stateful_object.py +29 -0
  45. loco_mujoco/core/terminal_state_handler/__init__.py +10 -0
  46. loco_mujoco/core/terminal_state_handler/base.py +199 -0
  47. loco_mujoco/core/terminal_state_handler/height.py +144 -0
  48. loco_mujoco/core/terminal_state_handler/no_terminal.py +105 -0
  49. loco_mujoco/core/terminal_state_handler/traj.py +307 -0
  50. loco_mujoco/core/terrain/__init__.py +11 -0
  51. loco_mujoco/core/terrain/base.py +336 -0
  52. loco_mujoco/core/terrain/dynamic.py +19 -0
  53. loco_mujoco/core/terrain/heightmap.py +469 -0
  54. loco_mujoco/core/terrain/rough.py +304 -0
  55. loco_mujoco/core/terrain/static.py +100 -0
  56. loco_mujoco/core/trajectory/__init__.py +10 -0
  57. loco_mujoco/core/trajectory/dataclasses.py +1317 -0
  58. loco_mujoco/core/trajectory/handler.py +394 -0
  59. loco_mujoco/core/utils/__init__.py +8 -0
  60. loco_mujoco/core/utils/backend.py +19 -0
  61. loco_mujoco/core/utils/decorators.py +12 -0
  62. loco_mujoco/core/utils/env.py +110 -0
  63. loco_mujoco/core/utils/flat_kinematics.py +102 -0
  64. loco_mujoco/core/utils/math.py +725 -0
  65. loco_mujoco/core/utils/models.py +196 -0
  66. loco_mujoco/core/utils/mujoco.py +175 -0
  67. loco_mujoco/core/visuals/__init__.py +5 -0
  68. loco_mujoco/core/visuals/scene.py +71 -0
  69. loco_mujoco/core/visuals/usd_exporter.py +218 -0
  70. loco_mujoco/core/visuals/video_recorder.py +104 -0
  71. loco_mujoco/core/visuals/viewer.py +904 -0
  72. loco_mujoco/core/visuals/viser_viewer.py +1059 -0
  73. loco_mujoco/core/wrappers/__init__.py +3 -0
  74. loco_mujoco/core/wrappers/gymnasium.py +190 -0
  75. loco_mujoco/core/wrappers/mjx.py +325 -0
  76. loco_mujoco/core/wrappers/rollout.py +146 -0
  77. loco_mujoco/datasets/data_generation/__init__.py +1 -0
  78. loco_mujoco/datasets/data_generation/utils.py +305 -0
  79. loco_mujoco/datasets/humanoids/LAFAN1/__init__.py +2 -0
  80. loco_mujoco/datasets/humanoids/LAFAN1/const.py +72 -0
  81. loco_mujoco/datasets/humanoids/LAFAN1/load.py +196 -0
  82. loco_mujoco/environments/__init__.py +3 -0
  83. loco_mujoco/environments/base.py +1045 -0
  84. loco_mujoco/environments/humanoids/__init__.py +60 -0
  85. loco_mujoco/environments/humanoids/apptronik_apollo.py +443 -0
  86. loco_mujoco/environments/humanoids/apptronik_apollo_mjx.py +47 -0
  87. loco_mujoco/environments/humanoids/atlas.py +420 -0
  88. loco_mujoco/environments/humanoids/atlas_mjx.py +43 -0
  89. loco_mujoco/environments/humanoids/base_robot_humanoid.py +62 -0
  90. loco_mujoco/environments/humanoids/base_skeleton.py +415 -0
  91. loco_mujoco/environments/humanoids/boostert1.py +373 -0
  92. loco_mujoco/environments/humanoids/boostert1_mjx.py +46 -0
  93. loco_mujoco/environments/humanoids/fourier_gr1t2.py +462 -0
  94. loco_mujoco/environments/humanoids/fourier_gr1t2_mjx.py +46 -0
  95. loco_mujoco/environments/humanoids/myoskeleton.py +798 -0
  96. loco_mujoco/environments/humanoids/myoskeleton_mjx.py +52 -0
  97. loco_mujoco/environments/humanoids/skeletons.py +746 -0
  98. loco_mujoco/environments/humanoids/talos.py +563 -0
  99. loco_mujoco/environments/humanoids/talos_mjx.py +41 -0
  100. loco_mujoco/environments/humanoids/toddlerbot.py +477 -0
  101. loco_mujoco/environments/humanoids/toddlerbot_mjx.py +45 -0
  102. loco_mujoco/environments/humanoids/unitreeG1.py +396 -0
  103. loco_mujoco/environments/humanoids/unitreeG1_mjx.py +47 -0
  104. loco_mujoco/environments/humanoids/unitreeH1.py +371 -0
  105. loco_mujoco/environments/humanoids/unitreeH1_mjx.py +70 -0
  106. loco_mujoco/environments/humanoids/unitreeH1v2.py +664 -0
  107. loco_mujoco/environments/humanoids/unitreeH1v2_mjx.py +55 -0
  108. loco_mujoco/environments/quadrupeds/__init__.py +22 -0
  109. loco_mujoco/environments/quadrupeds/anymal_c.py +305 -0
  110. loco_mujoco/environments/quadrupeds/anymal_c_mjx.py +48 -0
  111. loco_mujoco/environments/quadrupeds/base_robot_quadruped.py +17 -0
  112. loco_mujoco/environments/quadrupeds/bd_spot.py +335 -0
  113. loco_mujoco/environments/quadrupeds/bd_spot_mjx.py +48 -0
  114. loco_mujoco/environments/quadrupeds/unitreeA1.py +301 -0
  115. loco_mujoco/environments/quadrupeds/unitreeA1_mjx.py +49 -0
  116. loco_mujoco/environments/quadrupeds/unitreeGo2.py +324 -0
  117. loco_mujoco/environments/quadrupeds/unitreeGo2_mjx.py +48 -0
  118. loco_mujoco/smpl/__init__.py +5 -0
  119. loco_mujoco/smpl/const.py +189 -0
  120. loco_mujoco/smpl/generate_smplh_model.py +69 -0
  121. loco_mujoco/smpl/parser.py +555 -0
  122. loco_mujoco/smpl/retargeting.py +971 -0
  123. loco_mujoco/smpl/robot_confs/Apollo.yaml +6 -0
  124. loco_mujoco/smpl/robot_confs/Atlas.yaml +31 -0
  125. loco_mujoco/smpl/robot_confs/BoosterT1.yaml +4 -0
  126. loco_mujoco/smpl/robot_confs/FourierGR1T2.yaml +6 -0
  127. loco_mujoco/smpl/robot_confs/MyoSkeleton.yaml +9 -0
  128. loco_mujoco/smpl/robot_confs/SkeletonMuscle.yaml +11 -0
  129. loco_mujoco/smpl/robot_confs/SkeletonTorque.yaml +11 -0
  130. loco_mujoco/smpl/robot_confs/Talos.yaml +34 -0
  131. loco_mujoco/smpl/robot_confs/ToddlerBot.yaml +9 -0
  132. loco_mujoco/smpl/robot_confs/UnitreeG1.yaml +8 -0
  133. loco_mujoco/smpl/robot_confs/UnitreeH1.yaml +26 -0
  134. loco_mujoco/smpl/robot_confs/UnitreeH1v2.yaml +8 -0
  135. loco_mujoco/smpl/robot_confs/defaults.yaml +93 -0
  136. loco_mujoco/smpl/utils/smoothing.py +37 -0
  137. loco_mujoco/task_factories/__init__.py +11 -0
  138. loco_mujoco/task_factories/base.py +80 -0
  139. loco_mujoco/task_factories/dataset_confs.py +79 -0
  140. loco_mujoco/task_factories/imitation_factory.py +314 -0
  141. loco_mujoco/task_factories/rl_factory.py +52 -0
  142. loco_mujoco/utils/__init__.py +10 -0
  143. loco_mujoco/utils/dataset.py +356 -0
  144. loco_mujoco/utils/logging.py +36 -0
  145. loco_mujoco/utils/metrics.py +358 -0
  146. loco_mujoco/utils/myomodel_init.py +93 -0
  147. loco_mujoco/utils/retarget_for_env.py +184 -0
  148. loco_mujoco/utils/running_stats.py +134 -0
  149. loco_mujoco/utils/video.py +14 -0
  150. loco_mujoco-1.2.0.dist-info/METADATA +331 -0
  151. loco_mujoco-1.2.0.dist-info/RECORD +155 -0
  152. loco_mujoco-1.2.0.dist-info/WHEEL +5 -0
  153. loco_mujoco-1.2.0.dist-info/entry_points.txt +14 -0
  154. loco_mujoco-1.2.0.dist-info/licenses/LICENSE +21 -0
  155. loco_mujoco-1.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,50 @@
1
+ from pathlib import Path
2
+ import os
3
+ import yaml
4
+
5
+ __version__ = '1.2.0'
6
+
7
+
8
+ try:
9
+ PATH_TO_VARIABLES = Path(__file__).resolve().parent / "LOCOMUJOCO_VARIABLES.yaml"
10
+ PATH_TO_SMPL_ROBOT_CONF = Path(__file__).resolve().parent / "smpl" / "robot_confs"
11
+
12
+ # PATH_TO_CUSTOM_MODELS: used for MyoSkeleton and other custom models
13
+ # (e.g. loco-mujoco-myomodel-init). Resolved from LOCOMUJOCO_MODELS_PATH in the yaml.
14
+ def _resolve_path_to_custom_models() -> Path:
15
+ default_path = Path(os.path.expanduser("~")) / ".loco-mujoco-caches" / "models"
16
+ if not PATH_TO_VARIABLES.exists():
17
+ PATH_TO_VARIABLES.parent.mkdir(parents=True, exist_ok=True)
18
+ with open(PATH_TO_VARIABLES, "w") as f:
19
+ yaml.dump({"LOCOMUJOCO_MODELS_PATH": str(default_path)}, f, default_flow_style=False)
20
+ return default_path
21
+ with open(PATH_TO_VARIABLES) as f:
22
+ data = yaml.load(f, Loader=yaml.FullLoader) or {}
23
+ raw = data.get("LOCOMUJOCO_MODELS_PATH")
24
+ if isinstance(raw, list):
25
+ raw = raw[0] if raw else None
26
+ if isinstance(raw, str) and raw.strip():
27
+ return Path(os.path.expanduser(raw.strip()))
28
+ # Use default and persist it in the yaml so the path is always set there
29
+ data["LOCOMUJOCO_MODELS_PATH"] = str(default_path)
30
+ with open(PATH_TO_VARIABLES, "w") as f:
31
+ yaml.dump(data, f, default_flow_style=False)
32
+ return default_path
33
+
34
+ PATH_TO_CUSTOM_MODELS = _resolve_path_to_custom_models()
35
+
36
+ from .core import Mujoco, Mjx
37
+ from .environments import LocoEnv
38
+ from .task_factories import (TaskFactory, RLFactory, ImitationFactory)
39
+
40
+ def get_registered_envs():
41
+ return LocoEnv.registered_envs
42
+
43
+ def get_variable(key):
44
+ try:
45
+ return yaml.load(open(PATH_TO_VARIABLES), Loader=yaml.FullLoader)[key]
46
+ except KeyError:
47
+ return None
48
+
49
+ except ImportError as e:
50
+ print(e)
@@ -0,0 +1,7 @@
1
+ from .common import *
2
+ from loco_mujoco.algorithms.common.networks import FullyConnectedNet, ActorCritic, RunningMeanStd
3
+ from .ppo_jax import PPOJax
4
+ from .gail_jax import GAILJax
5
+ from .amp_jax import AMPJax
6
+ from . import experimental
7
+ from .supermimic_jax import SuperMimicJax
@@ -0,0 +1,30 @@
1
+ import jax.numpy as jnp
2
+
3
+ from loco_mujoco.algorithms import GAILJax
4
+
5
+
6
+ class AMPJax(GAILJax):
7
+
8
+ @classmethod
9
+ def _predict_rewards(cls, obs, discriminator, disc_train_state):
10
+ logits, _ = discriminator.apply({'params': disc_train_state.params,
11
+ 'run_stats': disc_train_state.run_stats},
12
+ obs, mutable=["run_stats"])
13
+
14
+ reward = jnp.maximum(0.0, 1 - 0.25*jnp.square(logits - 1))
15
+
16
+ return reward
17
+
18
+ @classmethod
19
+ def _discriminator_loss(cls, config, logits, targets):
20
+
21
+ # least squares loss
22
+ total_loss = jnp.mean(jnp.square(logits - targets))
23
+
24
+ return total_loss, logits
25
+
26
+ @classmethod
27
+ def _get_discriminator_targets(cls, plcy_batch_size, expert_batch_size):
28
+ plcy_target = -1 * jnp.ones(shape=(plcy_batch_size,))
29
+ expert_target = 1 * jnp.ones(shape=(expert_batch_size,))
30
+ return plcy_target, expert_target
@@ -0,0 +1,2 @@
1
+ from .base_algorithm import *
2
+ from .dataclasses import *
@@ -0,0 +1,159 @@
1
+ import pickle
2
+ from pathlib import Path
3
+ from dataclasses import dataclass
4
+
5
+ from flax import struct
6
+
7
+ from loco_mujoco.utils import MetricsHandler
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class AgentConfBase:
12
+ """
13
+ Abstract base class for *static* agent configuration.
14
+ Any subclass must implement the serialize and from_dict methods.
15
+ """
16
+
17
+ def serialize(self):
18
+ """Serialize the agent configuration."""
19
+ raise NotImplementedError
20
+
21
+ @classmethod
22
+ def from_dict(cls, d):
23
+ """Create an instance of the configuration class from a dictionary."""
24
+ raise NotImplementedError
25
+
26
+
27
+ @struct.dataclass
28
+ class AgentStateBase:
29
+ """
30
+ Abstract base class for *dynamic* agent state.
31
+ Any subclass must implement the serialize and from_dict methods.
32
+ """
33
+
34
+ def serialize(self):
35
+ raise NotImplementedError
36
+
37
+ @classmethod
38
+ def from_dict(cls, d, agent_conf):
39
+ raise NotImplementedError
40
+
41
+
42
+ class JaxRLAlgorithmBase:
43
+
44
+ """
45
+ Base class defining the interface for all JAX RL algorithms.
46
+ """
47
+
48
+ # these class attributes must be set by subclasses
49
+ _agent_conf: AgentConfBase
50
+ _agent_state: AgentStateBase
51
+
52
+ # prefix for all saved agent files
53
+ _saved_agent_suffix = ".pkl"
54
+
55
+ @classmethod
56
+ def init_agent_conf(cls, env, config):
57
+ """
58
+ Initialize the agent configuration.
59
+ """
60
+ raise NotImplementedError
61
+
62
+ @classmethod
63
+ def init_agent_state(cls, env, agent_conf: AgentConfBase, rng) -> AgentStateBase:
64
+ """ Initializes and returns the agent state (network params, optimizer state, etc.). """
65
+ raise NotImplementedError
66
+
67
+ @classmethod
68
+ def build_train_fn(cls, env, agent_conf: AgentConfBase, mh: MetricsHandler = None):
69
+ """ Returns the train function. Takes (rng, agent_state, traj) and runs one training chunk. """
70
+ return lambda rng_key, agent_state, traj: cls._train_fn(rng_key, env, agent_conf, agent_state, traj=traj, mh=mh)
71
+
72
+ @classmethod
73
+ def _train_fn(cls, rng, env,
74
+ agent_conf: AgentConfBase,
75
+ agent_state: AgentStateBase = None,
76
+ traj=None,
77
+ mh: MetricsHandler = None):
78
+ """ Main training algorithm of an RL algorithm. """
79
+ raise NotImplementedError
80
+
81
+ @classmethod
82
+ def build_eval_fn(cls, env, agent_conf: AgentConfBase, mh: MetricsHandler = None):
83
+ """ Returns the eval function. Same signature as build_train_fn:
84
+ takes (rng, agent_state, traj) and runs an evaluation rollout,
85
+ returning a dict with metric summaries. Does not mutate the agent's
86
+ training state (any local run_stats updates during eval are
87
+ discarded with the eval scan carry). """
88
+ return lambda rng_key, agent_state, traj: cls._eval_fn(rng_key, env, agent_conf, agent_state, traj=traj, mh=mh)
89
+
90
+ @classmethod
91
+ def _eval_fn(cls, rng, env,
92
+ agent_conf: AgentConfBase,
93
+ agent_state: AgentStateBase,
94
+ traj=None,
95
+ mh: MetricsHandler = None):
96
+ """ Evaluation rollout. Subclasses should implement this to run the
97
+ agent (greedy/deterministic if applicable) for a fixed number of
98
+ parallel env steps and return a metrics dict. """
99
+ raise NotImplementedError
100
+
101
+ @classmethod
102
+ def play_policy(cls, train_state, env, config, n_envs, n_steps=None, record=False, key=None):
103
+ raise NotImplementedError
104
+
105
+ @classmethod
106
+ def save_agent(cls, path, agent_conf: AgentConfBase, agent_state: AgentStateBase):
107
+ """ Save the agent state to a file."""
108
+ path = Path(path)
109
+ path = path / (cls.__name__ + "_saved")
110
+ path = path.with_suffix(cls._saved_agent_suffix)
111
+ # serialize agent state
112
+ serialized_state = cls.serialize(agent_conf, agent_state)
113
+ # save agent state
114
+ with open(path, 'wb') as file:
115
+ pickle.dump(serialized_state, file)
116
+ print(f"\nSaved agent to: {path}\n")
117
+ return path
118
+
119
+ @classmethod
120
+ def load_agent(cls, path):
121
+ """ Load the agent state from a file. """
122
+ if isinstance(path, str):
123
+ path = Path(path)
124
+ if not path.is_file():
125
+ raise ValueError(f'Not a file: {path}')
126
+ if path.suffix != cls._saved_agent_suffix:
127
+ raise ValueError(f'Not a {cls._saved_agent_suffix} file: {path}')
128
+ with open(path, 'rb') as file:
129
+ data = pickle.load(file)
130
+ return cls.from_dict(data)
131
+
132
+ @classmethod
133
+ def serialize(cls, agent_conf: AgentConfBase, agent_state: AgentStateBase):
134
+ """ Serialize conf and state of an agent. """
135
+ serialized_agent_conf = agent_conf.serialize()
136
+ serialized_agent_state = agent_state.serialize()
137
+ return {"agent_conf": serialized_agent_conf,
138
+ "agent_state": serialized_agent_state}
139
+
140
+ @classmethod
141
+ def from_dict(cls, d):
142
+ """ Load conf and state of an agent from a dictionary. """
143
+ agent_conf = cls._agent_conf.from_dict(d["agent_conf"])
144
+ agent_state = cls._agent_state.from_dict(d["agent_state"], agent_conf)
145
+ return agent_conf, agent_state
146
+
147
+ @staticmethod
148
+ def _wrap_env(env, config):
149
+ """ Wrap the environment with the necessary wrappers. """
150
+ raise NotImplementedError
151
+
152
+ @classmethod
153
+ def _linear_lr_schedule(cls, count, num_minibatches, update_epochs, lr, num_updates):
154
+ frac = (
155
+ 1.0
156
+ - (count // (num_minibatches * update_epochs))
157
+ / num_updates
158
+ )
159
+ return lr * frac
@@ -0,0 +1,381 @@
1
+ from typing import NamedTuple
2
+ from typing import Any
3
+
4
+ import jax
5
+ import jax.numpy as jnp
6
+ from flax import struct
7
+ from flax.training import train_state
8
+
9
+ from loco_mujoco.environments.base import TrajState
10
+ from loco_mujoco.core.wrappers.mjx import Metrics
11
+
12
+
13
+ class Transition(NamedTuple):
14
+ done: jnp.ndarray
15
+ absorbing: jnp.ndarray
16
+ action: jnp.ndarray
17
+ value: jnp.ndarray
18
+ reward: jnp.ndarray
19
+ log_prob: jnp.ndarray
20
+ obs: jnp.ndarray
21
+ info: jnp.ndarray
22
+ traj_state: TrajState
23
+ metrics: Metrics
24
+
25
+
26
+ class MetricHandlerTransition(NamedTuple):
27
+ env_state: Any
28
+ logged_metrics: Metrics
29
+
30
+
31
+ class TrainState(train_state.TrainState):
32
+ run_stats: Any
33
+
34
+
35
+ @struct.dataclass
36
+ class TrainStateBuffer:
37
+ train_states: TrainState
38
+ n: int
39
+ size: int # buffer size
40
+
41
+ @classmethod
42
+ def create(cls, train_state: TrainState, size: int):
43
+ return TrainStateBuffer(
44
+ train_states=jax.tree.map(lambda x: jnp.stack([x] * size), train_state),
45
+ n=0,
46
+ size=size
47
+ )
48
+
49
+ @classmethod
50
+ def add(cls, train_state_buffer, train_state: TrainState):
51
+ index = train_state_buffer.n
52
+ # Add the new train state at index n
53
+ train_states_updated = jax.tree.map(
54
+ lambda buffer, new: buffer.at[index].set(new),
55
+ train_state_buffer.train_states,
56
+ train_state
57
+ )
58
+ return train_state_buffer.replace(
59
+ train_states=train_states_updated,
60
+ n=index + 1,
61
+ )
62
+
63
+
64
+ @struct.dataclass
65
+ class RewardNormStats:
66
+ """
67
+ Running statistics for PPO-style reward normalization. Lives on the agent
68
+ state so it survives env resets and is saved with the agent.
69
+ """
70
+ mean: jnp.ndarray
71
+ var: jnp.ndarray
72
+ count: jnp.ndarray
73
+ return_val: jnp.ndarray
74
+
75
+ @classmethod
76
+ def create(cls, num_envs: int):
77
+ return cls(
78
+ mean=jnp.zeros(()),
79
+ var=jnp.ones(()),
80
+ count=jnp.full((), 1e-4),
81
+ return_val=jnp.zeros((num_envs,)),
82
+ )
83
+
84
+
85
+ def update_reward_norm(stats: "RewardNormStats", reward: jnp.ndarray,
86
+ done: jnp.ndarray, gamma: float):
87
+ """
88
+ Update running stats with a batch of rewards and return the normalized
89
+ reward plus the new stats. Mirrors the previous NormalizeVecReward wrapper.
90
+ """
91
+ return_val = stats.return_val * gamma * (1.0 - done) + reward
92
+
93
+ batch_mean = jnp.mean(return_val, axis=0)
94
+ batch_var = jnp.var(return_val, axis=0)
95
+ batch_count = return_val.shape[0]
96
+
97
+ delta = batch_mean - stats.mean
98
+ tot_count = stats.count + batch_count
99
+
100
+ new_mean = stats.mean + delta * batch_count / tot_count
101
+ m_a = stats.var * stats.count
102
+ m_b = batch_var * batch_count
103
+ M2 = m_a + m_b + jnp.square(delta) * stats.count * batch_count / tot_count
104
+ new_var = M2 / tot_count
105
+
106
+ new_stats = RewardNormStats(
107
+ mean=new_mean,
108
+ var=new_var,
109
+ count=tot_count,
110
+ return_val=return_val,
111
+ )
112
+ normalized_reward = reward / jnp.sqrt(new_stats.var + 1e-8)
113
+ return normalized_reward, new_stats
114
+
115
+
116
+ @struct.dataclass
117
+ class ReplayBuffer:
118
+ """Simple circular replay buffer for off-policy / imitation algorithms.
119
+
120
+ Core columns: (obs, next_obs, action, reward, done). `value_target` is
121
+ an optional column used by value-distillation-style critic learning
122
+ (e.g. VanillaDagger stores `V_teacher(s)` there at collect time).
123
+
124
+ Any of `next_obs` / `value_target` can be turned off at `create` time;
125
+ disabled columns become zero-sized dummies (0 bytes). Flags are static
126
+ metadata so `if self.store_*` branches are baked at trace time.
127
+
128
+ Optional Bernoulli sub-sampling at write time: with `add_subsample=True`
129
+ and `add_prob=p`, each transition in an incoming batch is kept with
130
+ probability p. Effective temporal coverage becomes ~capacity / p without
131
+ growing GPU memory. `add_batch` requires an `rng` when subsampling is on.
132
+ """
133
+ obs: jnp.ndarray
134
+ next_obs: jnp.ndarray
135
+ action: jnp.ndarray
136
+ reward: jnp.ndarray
137
+ done: jnp.ndarray
138
+ value_target: jnp.ndarray
139
+ ptr: int
140
+ size: int
141
+ store_next_obs: bool = struct.field(pytree_node=False, default=True)
142
+ store_value_target: bool = struct.field(pytree_node=False, default=False)
143
+ add_subsample: bool = struct.field(pytree_node=False, default=False)
144
+ add_prob: float = struct.field(pytree_node=False, default=1.0)
145
+
146
+ @classmethod
147
+ def create(cls, obs_dim: int, action_dim: int, capacity: int,
148
+ store_next_obs: bool = True,
149
+ store_value_target: bool = False,
150
+ add_subsample: bool = False,
151
+ add_prob: float = 1.0):
152
+ next_obs_shape = (capacity, obs_dim) if store_next_obs else (0, 0)
153
+ value_shape = (capacity,) if store_value_target else (0,)
154
+ return cls(
155
+ obs=jnp.zeros((capacity, obs_dim)),
156
+ next_obs=jnp.zeros(next_obs_shape),
157
+ action=jnp.zeros((capacity, action_dim)),
158
+ reward=jnp.zeros((capacity,)),
159
+ done=jnp.zeros((capacity,), dtype=jnp.float32),
160
+ value_target=jnp.zeros(value_shape),
161
+ ptr=0,
162
+ size=0,
163
+ store_next_obs=store_next_obs,
164
+ store_value_target=store_value_target,
165
+ add_subsample=bool(add_subsample),
166
+ add_prob=float(add_prob),
167
+ )
168
+
169
+ def add_batch(self, obs, next_obs, action, reward, done,
170
+ value_target=None, rng=None):
171
+ capacity = self.obs.shape[0]
172
+ batch_size = obs.shape[0]
173
+
174
+ if self.add_subsample:
175
+ assert rng is not None, (
176
+ "buffer was created with add_subsample=True but "
177
+ "add_batch received rng=None"
178
+ )
179
+ # Per-element Bernoulli keep mask.
180
+ keep = jax.random.uniform(rng, (batch_size,)) < self.add_prob
181
+ kept_cnt = jnp.sum(keep.astype(jnp.int32))
182
+ # Kept entries are written to consecutive slots; dropped entries get the
183
+ # out-of-range index `capacity`, which mode="drop" discards. They must not be
184
+ # routed to a real slot: a scatter with duplicate indices applies its updates
185
+ # in an unspecified order, so a dropped entry could overwrite a kept one.
186
+ offsets = jnp.cumsum(keep.astype(jnp.int32)) - 1
187
+ indices = jnp.where(keep, (self.ptr + offsets) % capacity, capacity)
188
+ if self.store_next_obs:
189
+ new_next_obs = self.next_obs.at[indices].set(next_obs, mode="drop")
190
+ else:
191
+ new_next_obs = self.next_obs
192
+ if self.store_value_target:
193
+ assert value_target is not None, (
194
+ "buffer was created with store_value_target=True but "
195
+ "add_batch received value_target=None"
196
+ )
197
+ new_value_target = self.value_target.at[indices].set(value_target, mode="drop")
198
+ else:
199
+ new_value_target = self.value_target
200
+ return self.replace(
201
+ obs=self.obs.at[indices].set(obs, mode="drop"),
202
+ next_obs=new_next_obs,
203
+ action=self.action.at[indices].set(action, mode="drop"),
204
+ reward=self.reward.at[indices].set(reward, mode="drop"),
205
+ done=self.done.at[indices].set(done.astype(jnp.float32), mode="drop"),
206
+ value_target=new_value_target,
207
+ ptr=(self.ptr + kept_cnt) % capacity,
208
+ size=jnp.minimum(self.size + kept_cnt, capacity),
209
+ )
210
+
211
+ indices = (self.ptr + jnp.arange(batch_size)) % capacity
212
+ new_next_obs = (self.next_obs.at[indices].set(next_obs)
213
+ if self.store_next_obs else self.next_obs)
214
+ if self.store_value_target:
215
+ assert value_target is not None, (
216
+ "buffer was created with store_value_target=True but "
217
+ "add_batch received value_target=None"
218
+ )
219
+ new_value_target = self.value_target.at[indices].set(value_target)
220
+ else:
221
+ new_value_target = self.value_target
222
+ return self.replace(
223
+ obs=self.obs.at[indices].set(obs),
224
+ next_obs=new_next_obs,
225
+ action=self.action.at[indices].set(action),
226
+ reward=self.reward.at[indices].set(reward),
227
+ done=self.done.at[indices].set(done.astype(jnp.float32)),
228
+ value_target=new_value_target,
229
+ ptr=(self.ptr + batch_size) % capacity,
230
+ size=jnp.minimum(self.size + batch_size, capacity),
231
+ )
232
+
233
+ def sample(self, rng, batch_size: int):
234
+ indices = jax.random.randint(rng, (batch_size,), 0, self.size)
235
+ if self.store_next_obs:
236
+ next_obs = self.next_obs[indices]
237
+ else:
238
+ next_obs = jnp.zeros((batch_size, 0), dtype=self.obs.dtype)
239
+ return (
240
+ self.obs[indices],
241
+ next_obs,
242
+ self.action[indices],
243
+ self.reward[indices],
244
+ self.done[indices],
245
+ )
246
+
247
+
248
+ @struct.dataclass
249
+ class ReservoirBuffer:
250
+ """Reservoir-sampled long-term memory.
251
+
252
+ Approximates a uniform sample of all transitions ever passed through
253
+ `add_batch`. Capacity stays fixed; early additions fill sequentially,
254
+ later additions probabilistically replace random slots. At steady
255
+ state each global transition has probability `capacity / total_seen`
256
+ of being in the buffer.
257
+
258
+ Columns mirror `ReplayBuffer` (obs / action / value_target). No
259
+ `next_obs` or `reward` / `done` — this is intended as a
260
+ long-term BC + value-distillation store, not a TD buffer.
261
+
262
+ `min_include_prob` optionally floors the per-item inclusion rate. The
263
+ pure reservoir's rate decays as k/n indefinitely, which is correct
264
+ for stationary streams but wastes signal when the data distribution
265
+ drifts (e.g. DAgger rollouts shift as the student learns). Setting
266
+ `min_include_prob > 0` caps the effective n at `ceil(k / p_min)`, so
267
+ the buffer behaves like a uniform sample over the last `k / p_min`
268
+ transitions once that window has filled. Default 0.0 = pure reservoir.
269
+ """
270
+ obs: jnp.ndarray
271
+ action: jnp.ndarray
272
+ value_target: jnp.ndarray
273
+ total_seen: jnp.ndarray # scalar int32, counts all transitions ever added
274
+ size: jnp.ndarray # scalar int32, min(total_seen, capacity)
275
+ store_value_target: bool = struct.field(pytree_node=False, default=False)
276
+ min_include_prob: float = struct.field(pytree_node=False, default=0.0)
277
+
278
+ @classmethod
279
+ def create(cls, obs_dim: int, action_dim: int, capacity: int,
280
+ store_value_target: bool = False,
281
+ min_include_prob: float = 0.0):
282
+ value_shape = (capacity,) if store_value_target else (0,)
283
+ return cls(
284
+ obs=jnp.zeros((capacity, obs_dim)),
285
+ action=jnp.zeros((capacity, action_dim)),
286
+ value_target=jnp.zeros(value_shape),
287
+ total_seen=jnp.zeros((), dtype=jnp.int32),
288
+ size=jnp.zeros((), dtype=jnp.int32),
289
+ store_value_target=store_value_target,
290
+ min_include_prob=float(min_include_prob),
291
+ )
292
+
293
+ def add_batch(self, obs, action, value_target, rng):
294
+ """Reservoir-insert a batch. Vectorised: for each element i with
295
+ global index g_i = total_seen + i, draw j_i ~ uniform_int[0, n_eff_i)
296
+ where `n_eff_i = min(g_i+1, ceil(capacity / min_include_prob))`.
297
+ Write at slot g_i if we're still in the fill phase, else write at
298
+ j_i when j_i < capacity. Collisions between two include=True
299
+ elements writing to the same slot are resolved by whichever wins
300
+ the scatter — a tiny bias but costs nothing.
301
+ """
302
+ import math
303
+ capacity = self.obs.shape[0]
304
+ B = obs.shape[0]
305
+ batch_idx = self.total_seen + jnp.arange(B, dtype=jnp.int32)
306
+
307
+ # Effective denominator for the reservoir decision. A positive
308
+ # min_include_prob caps it at ceil(k/p_min), flooring the
309
+ # inclusion probability at p_min. Python-level branch — static.
310
+ if self.min_include_prob > 0.0:
311
+ cap_n = int(math.ceil(capacity / self.min_include_prob))
312
+ n_eff = jnp.minimum(batch_idx + 1, jnp.int32(cap_n))
313
+ else:
314
+ n_eff = batch_idx + 1
315
+
316
+ u = jax.random.uniform(rng, (B,))
317
+ j = jnp.floor(u * n_eff.astype(jnp.float32)).astype(jnp.int32)
318
+
319
+ fill = batch_idx < capacity
320
+ include = fill | (j < capacity)
321
+ slot = jnp.where(fill, batch_idx, j)
322
+ slot = jnp.clip(slot, 0, capacity - 1)
323
+
324
+ # Masked scatter: for non-include entries, write back the current
325
+ # slot contents (no-op). Collisions between include=True entries
326
+ # are rare for capacity >> batch_size.
327
+ fresh_obs = jnp.where(include[:, None], obs, self.obs[slot])
328
+ fresh_action = jnp.where(include[:, None], action, self.action[slot])
329
+ new_obs = self.obs.at[slot].set(fresh_obs)
330
+ new_action = self.action.at[slot].set(fresh_action)
331
+
332
+ if self.store_value_target:
333
+ assert value_target is not None, (
334
+ "reservoir was created with store_value_target=True but "
335
+ "add_batch received value_target=None"
336
+ )
337
+ fresh_vt = jnp.where(include, value_target, self.value_target[slot])
338
+ new_vt = self.value_target.at[slot].set(fresh_vt)
339
+ else:
340
+ new_vt = self.value_target
341
+
342
+ new_total = self.total_seen + B
343
+ new_size = jnp.minimum(jnp.int32(capacity), new_total)
344
+ return self.replace(
345
+ obs=new_obs,
346
+ action=new_action,
347
+ value_target=new_vt,
348
+ total_seen=new_total,
349
+ size=new_size,
350
+ )
351
+
352
+ def sample(self, rng, batch_size: int):
353
+ indices = jax.random.randint(rng, (batch_size,), 0, jnp.maximum(1, self.size))
354
+ return (
355
+ self.obs[indices],
356
+ self.action[indices],
357
+ (self.value_target[indices] if self.store_value_target
358
+ else jnp.zeros((batch_size,), dtype=self.obs.dtype)),
359
+ )
360
+
361
+
362
+ @struct.dataclass
363
+ class BestTrainStates:
364
+ train_states: TrainState
365
+ metrics: jnp.array
366
+ iterations: jnp.array
367
+ cur_worst_perf: float
368
+ step: int
369
+ n: int
370
+ size: int
371
+
372
+ @classmethod
373
+ def create(cls, train_state: TrainState, n: int):
374
+ return BestTrainStates(
375
+ train_states=jax.tree.map(lambda x: jnp.stack([x] * n), train_state),
376
+ metrics=jnp.full((n,), -jnp.inf),
377
+ iterations=jnp.zeros((n,)),
378
+ cur_worst_perf=-jnp.inf,
379
+ n=n,
380
+ size=0
381
+ )