loco-mujoco 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loco_mujoco/__init__.py +50 -0
- loco_mujoco/algorithms/__init__.py +7 -0
- loco_mujoco/algorithms/amp_jax.py +30 -0
- loco_mujoco/algorithms/common/__init__.py +2 -0
- loco_mujoco/algorithms/common/base_algorithm.py +159 -0
- loco_mujoco/algorithms/common/dataclasses.py +381 -0
- loco_mujoco/algorithms/common/networks.py +125 -0
- loco_mujoco/algorithms/common/policies.py +81 -0
- loco_mujoco/algorithms/experimental/__init__.py +7 -0
- loco_mujoco/algorithms/experimental/bptt_ppo_jax.py +818 -0
- loco_mujoco/algorithms/experimental/history_ppo_jax.py +732 -0
- loco_mujoco/algorithms/experimental/offpolicy_base.py +781 -0
- loco_mujoco/algorithms/experimental/s2pg_ppo_jax.py +800 -0
- loco_mujoco/algorithms/experimental/sac_jax.py +508 -0
- loco_mujoco/algorithms/experimental/td3_jax.py +1008 -0
- loco_mujoco/algorithms/experimental/vanilla_dagger_jax.py +889 -0
- loco_mujoco/algorithms/gail_jax.py +728 -0
- loco_mujoco/algorithms/ppo_jax.py +759 -0
- loco_mujoco/algorithms/supermimic_jax.py +1767 -0
- loco_mujoco/core/__init__.py +10 -0
- loco_mujoco/core/control_functions/__init__.py +7 -0
- loco_mujoco/core/control_functions/base.py +169 -0
- loco_mujoco/core/control_functions/default.py +77 -0
- loco_mujoco/core/control_functions/pd.py +191 -0
- loco_mujoco/core/domain_randomizer/__init__.py +7 -0
- loco_mujoco/core/domain_randomizer/base.py +205 -0
- loco_mujoco/core/domain_randomizer/default.py +842 -0
- loco_mujoco/core/domain_randomizer/no_randomization.py +108 -0
- loco_mujoco/core/initial_state_handler/__init__.py +7 -0
- loco_mujoco/core/initial_state_handler/base.py +84 -0
- loco_mujoco/core/initial_state_handler/default.py +110 -0
- loco_mujoco/core/initial_state_handler/traj_init_state.py +123 -0
- loco_mujoco/core/mujoco_base.py +1607 -0
- loco_mujoco/core/mujoco_mjx.py +835 -0
- loco_mujoco/core/observations/__init__.py +13 -0
- loco_mujoco/core/observations/base.py +1190 -0
- loco_mujoco/core/observations/goals.py +1217 -0
- loco_mujoco/core/observations/visualizer.py +213 -0
- loco_mujoco/core/reward/__init__.py +13 -0
- loco_mujoco/core/reward/base.py +126 -0
- loco_mujoco/core/reward/default.py +519 -0
- loco_mujoco/core/reward/trajectory_based.py +648 -0
- loco_mujoco/core/reward/utils.py +40 -0
- loco_mujoco/core/stateful_object.py +29 -0
- loco_mujoco/core/terminal_state_handler/__init__.py +10 -0
- loco_mujoco/core/terminal_state_handler/base.py +199 -0
- loco_mujoco/core/terminal_state_handler/height.py +144 -0
- loco_mujoco/core/terminal_state_handler/no_terminal.py +105 -0
- loco_mujoco/core/terminal_state_handler/traj.py +307 -0
- loco_mujoco/core/terrain/__init__.py +11 -0
- loco_mujoco/core/terrain/base.py +336 -0
- loco_mujoco/core/terrain/dynamic.py +19 -0
- loco_mujoco/core/terrain/heightmap.py +469 -0
- loco_mujoco/core/terrain/rough.py +304 -0
- loco_mujoco/core/terrain/static.py +100 -0
- loco_mujoco/core/trajectory/__init__.py +10 -0
- loco_mujoco/core/trajectory/dataclasses.py +1317 -0
- loco_mujoco/core/trajectory/handler.py +394 -0
- loco_mujoco/core/utils/__init__.py +8 -0
- loco_mujoco/core/utils/backend.py +19 -0
- loco_mujoco/core/utils/decorators.py +12 -0
- loco_mujoco/core/utils/env.py +110 -0
- loco_mujoco/core/utils/flat_kinematics.py +102 -0
- loco_mujoco/core/utils/math.py +725 -0
- loco_mujoco/core/utils/models.py +196 -0
- loco_mujoco/core/utils/mujoco.py +175 -0
- loco_mujoco/core/visuals/__init__.py +5 -0
- loco_mujoco/core/visuals/scene.py +71 -0
- loco_mujoco/core/visuals/usd_exporter.py +218 -0
- loco_mujoco/core/visuals/video_recorder.py +104 -0
- loco_mujoco/core/visuals/viewer.py +904 -0
- loco_mujoco/core/visuals/viser_viewer.py +1059 -0
- loco_mujoco/core/wrappers/__init__.py +3 -0
- loco_mujoco/core/wrappers/gymnasium.py +190 -0
- loco_mujoco/core/wrappers/mjx.py +325 -0
- loco_mujoco/core/wrappers/rollout.py +146 -0
- loco_mujoco/datasets/data_generation/__init__.py +1 -0
- loco_mujoco/datasets/data_generation/utils.py +305 -0
- loco_mujoco/datasets/humanoids/LAFAN1/__init__.py +2 -0
- loco_mujoco/datasets/humanoids/LAFAN1/const.py +72 -0
- loco_mujoco/datasets/humanoids/LAFAN1/load.py +196 -0
- loco_mujoco/environments/__init__.py +3 -0
- loco_mujoco/environments/base.py +1045 -0
- loco_mujoco/environments/humanoids/__init__.py +60 -0
- loco_mujoco/environments/humanoids/apptronik_apollo.py +443 -0
- loco_mujoco/environments/humanoids/apptronik_apollo_mjx.py +47 -0
- loco_mujoco/environments/humanoids/atlas.py +420 -0
- loco_mujoco/environments/humanoids/atlas_mjx.py +43 -0
- loco_mujoco/environments/humanoids/base_robot_humanoid.py +62 -0
- loco_mujoco/environments/humanoids/base_skeleton.py +415 -0
- loco_mujoco/environments/humanoids/boostert1.py +373 -0
- loco_mujoco/environments/humanoids/boostert1_mjx.py +46 -0
- loco_mujoco/environments/humanoids/fourier_gr1t2.py +462 -0
- loco_mujoco/environments/humanoids/fourier_gr1t2_mjx.py +46 -0
- loco_mujoco/environments/humanoids/myoskeleton.py +798 -0
- loco_mujoco/environments/humanoids/myoskeleton_mjx.py +52 -0
- loco_mujoco/environments/humanoids/skeletons.py +746 -0
- loco_mujoco/environments/humanoids/talos.py +563 -0
- loco_mujoco/environments/humanoids/talos_mjx.py +41 -0
- loco_mujoco/environments/humanoids/toddlerbot.py +477 -0
- loco_mujoco/environments/humanoids/toddlerbot_mjx.py +45 -0
- loco_mujoco/environments/humanoids/unitreeG1.py +396 -0
- loco_mujoco/environments/humanoids/unitreeG1_mjx.py +47 -0
- loco_mujoco/environments/humanoids/unitreeH1.py +371 -0
- loco_mujoco/environments/humanoids/unitreeH1_mjx.py +70 -0
- loco_mujoco/environments/humanoids/unitreeH1v2.py +664 -0
- loco_mujoco/environments/humanoids/unitreeH1v2_mjx.py +55 -0
- loco_mujoco/environments/quadrupeds/__init__.py +22 -0
- loco_mujoco/environments/quadrupeds/anymal_c.py +305 -0
- loco_mujoco/environments/quadrupeds/anymal_c_mjx.py +48 -0
- loco_mujoco/environments/quadrupeds/base_robot_quadruped.py +17 -0
- loco_mujoco/environments/quadrupeds/bd_spot.py +335 -0
- loco_mujoco/environments/quadrupeds/bd_spot_mjx.py +48 -0
- loco_mujoco/environments/quadrupeds/unitreeA1.py +301 -0
- loco_mujoco/environments/quadrupeds/unitreeA1_mjx.py +49 -0
- loco_mujoco/environments/quadrupeds/unitreeGo2.py +324 -0
- loco_mujoco/environments/quadrupeds/unitreeGo2_mjx.py +48 -0
- loco_mujoco/smpl/__init__.py +5 -0
- loco_mujoco/smpl/const.py +189 -0
- loco_mujoco/smpl/generate_smplh_model.py +69 -0
- loco_mujoco/smpl/parser.py +555 -0
- loco_mujoco/smpl/retargeting.py +971 -0
- loco_mujoco/smpl/robot_confs/Apollo.yaml +6 -0
- loco_mujoco/smpl/robot_confs/Atlas.yaml +31 -0
- loco_mujoco/smpl/robot_confs/BoosterT1.yaml +4 -0
- loco_mujoco/smpl/robot_confs/FourierGR1T2.yaml +6 -0
- loco_mujoco/smpl/robot_confs/MyoSkeleton.yaml +9 -0
- loco_mujoco/smpl/robot_confs/SkeletonMuscle.yaml +11 -0
- loco_mujoco/smpl/robot_confs/SkeletonTorque.yaml +11 -0
- loco_mujoco/smpl/robot_confs/Talos.yaml +34 -0
- loco_mujoco/smpl/robot_confs/ToddlerBot.yaml +9 -0
- loco_mujoco/smpl/robot_confs/UnitreeG1.yaml +8 -0
- loco_mujoco/smpl/robot_confs/UnitreeH1.yaml +26 -0
- loco_mujoco/smpl/robot_confs/UnitreeH1v2.yaml +8 -0
- loco_mujoco/smpl/robot_confs/defaults.yaml +93 -0
- loco_mujoco/smpl/utils/smoothing.py +37 -0
- loco_mujoco/task_factories/__init__.py +11 -0
- loco_mujoco/task_factories/base.py +80 -0
- loco_mujoco/task_factories/dataset_confs.py +79 -0
- loco_mujoco/task_factories/imitation_factory.py +314 -0
- loco_mujoco/task_factories/rl_factory.py +52 -0
- loco_mujoco/utils/__init__.py +10 -0
- loco_mujoco/utils/dataset.py +356 -0
- loco_mujoco/utils/logging.py +36 -0
- loco_mujoco/utils/metrics.py +358 -0
- loco_mujoco/utils/myomodel_init.py +93 -0
- loco_mujoco/utils/retarget_for_env.py +184 -0
- loco_mujoco/utils/running_stats.py +134 -0
- loco_mujoco/utils/video.py +14 -0
- loco_mujoco-1.2.0.dist-info/METADATA +331 -0
- loco_mujoco-1.2.0.dist-info/RECORD +155 -0
- loco_mujoco-1.2.0.dist-info/WHEEL +5 -0
- loco_mujoco-1.2.0.dist-info/entry_points.txt +14 -0
- loco_mujoco-1.2.0.dist-info/licenses/LICENSE +21 -0
- loco_mujoco-1.2.0.dist-info/top_level.txt +1 -0
loco_mujoco/__init__.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import os
|
|
3
|
+
import yaml
|
|
4
|
+
|
|
5
|
+
__version__ = '1.2.0'
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
PATH_TO_VARIABLES = Path(__file__).resolve().parent / "LOCOMUJOCO_VARIABLES.yaml"
|
|
10
|
+
PATH_TO_SMPL_ROBOT_CONF = Path(__file__).resolve().parent / "smpl" / "robot_confs"
|
|
11
|
+
|
|
12
|
+
# PATH_TO_CUSTOM_MODELS: used for MyoSkeleton and other custom models
|
|
13
|
+
# (e.g. loco-mujoco-myomodel-init). Resolved from LOCOMUJOCO_MODELS_PATH in the yaml.
|
|
14
|
+
def _resolve_path_to_custom_models() -> Path:
|
|
15
|
+
default_path = Path(os.path.expanduser("~")) / ".loco-mujoco-caches" / "models"
|
|
16
|
+
if not PATH_TO_VARIABLES.exists():
|
|
17
|
+
PATH_TO_VARIABLES.parent.mkdir(parents=True, exist_ok=True)
|
|
18
|
+
with open(PATH_TO_VARIABLES, "w") as f:
|
|
19
|
+
yaml.dump({"LOCOMUJOCO_MODELS_PATH": str(default_path)}, f, default_flow_style=False)
|
|
20
|
+
return default_path
|
|
21
|
+
with open(PATH_TO_VARIABLES) as f:
|
|
22
|
+
data = yaml.load(f, Loader=yaml.FullLoader) or {}
|
|
23
|
+
raw = data.get("LOCOMUJOCO_MODELS_PATH")
|
|
24
|
+
if isinstance(raw, list):
|
|
25
|
+
raw = raw[0] if raw else None
|
|
26
|
+
if isinstance(raw, str) and raw.strip():
|
|
27
|
+
return Path(os.path.expanduser(raw.strip()))
|
|
28
|
+
# Use default and persist it in the yaml so the path is always set there
|
|
29
|
+
data["LOCOMUJOCO_MODELS_PATH"] = str(default_path)
|
|
30
|
+
with open(PATH_TO_VARIABLES, "w") as f:
|
|
31
|
+
yaml.dump(data, f, default_flow_style=False)
|
|
32
|
+
return default_path
|
|
33
|
+
|
|
34
|
+
PATH_TO_CUSTOM_MODELS = _resolve_path_to_custom_models()
|
|
35
|
+
|
|
36
|
+
from .core import Mujoco, Mjx
|
|
37
|
+
from .environments import LocoEnv
|
|
38
|
+
from .task_factories import (TaskFactory, RLFactory, ImitationFactory)
|
|
39
|
+
|
|
40
|
+
def get_registered_envs():
|
|
41
|
+
return LocoEnv.registered_envs
|
|
42
|
+
|
|
43
|
+
def get_variable(key):
|
|
44
|
+
try:
|
|
45
|
+
return yaml.load(open(PATH_TO_VARIABLES), Loader=yaml.FullLoader)[key]
|
|
46
|
+
except KeyError:
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
except ImportError as e:
|
|
50
|
+
print(e)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
from .common import *
|
|
2
|
+
from loco_mujoco.algorithms.common.networks import FullyConnectedNet, ActorCritic, RunningMeanStd
|
|
3
|
+
from .ppo_jax import PPOJax
|
|
4
|
+
from .gail_jax import GAILJax
|
|
5
|
+
from .amp_jax import AMPJax
|
|
6
|
+
from . import experimental
|
|
7
|
+
from .supermimic_jax import SuperMimicJax
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import jax.numpy as jnp
|
|
2
|
+
|
|
3
|
+
from loco_mujoco.algorithms import GAILJax
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AMPJax(GAILJax):
|
|
7
|
+
|
|
8
|
+
@classmethod
|
|
9
|
+
def _predict_rewards(cls, obs, discriminator, disc_train_state):
|
|
10
|
+
logits, _ = discriminator.apply({'params': disc_train_state.params,
|
|
11
|
+
'run_stats': disc_train_state.run_stats},
|
|
12
|
+
obs, mutable=["run_stats"])
|
|
13
|
+
|
|
14
|
+
reward = jnp.maximum(0.0, 1 - 0.25*jnp.square(logits - 1))
|
|
15
|
+
|
|
16
|
+
return reward
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def _discriminator_loss(cls, config, logits, targets):
|
|
20
|
+
|
|
21
|
+
# least squares loss
|
|
22
|
+
total_loss = jnp.mean(jnp.square(logits - targets))
|
|
23
|
+
|
|
24
|
+
return total_loss, logits
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def _get_discriminator_targets(cls, plcy_batch_size, expert_batch_size):
|
|
28
|
+
plcy_target = -1 * jnp.ones(shape=(plcy_batch_size,))
|
|
29
|
+
expert_target = 1 * jnp.ones(shape=(expert_batch_size,))
|
|
30
|
+
return plcy_target, expert_target
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
from flax import struct
|
|
6
|
+
|
|
7
|
+
from loco_mujoco.utils import MetricsHandler
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class AgentConfBase:
|
|
12
|
+
"""
|
|
13
|
+
Abstract base class for *static* agent configuration.
|
|
14
|
+
Any subclass must implement the serialize and from_dict methods.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def serialize(self):
|
|
18
|
+
"""Serialize the agent configuration."""
|
|
19
|
+
raise NotImplementedError
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def from_dict(cls, d):
|
|
23
|
+
"""Create an instance of the configuration class from a dictionary."""
|
|
24
|
+
raise NotImplementedError
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@struct.dataclass
|
|
28
|
+
class AgentStateBase:
|
|
29
|
+
"""
|
|
30
|
+
Abstract base class for *dynamic* agent state.
|
|
31
|
+
Any subclass must implement the serialize and from_dict methods.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def serialize(self):
|
|
35
|
+
raise NotImplementedError
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_dict(cls, d, agent_conf):
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class JaxRLAlgorithmBase:
|
|
43
|
+
|
|
44
|
+
"""
|
|
45
|
+
Base class defining the interface for all JAX RL algorithms.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# these class attributes must be set by subclasses
|
|
49
|
+
_agent_conf: AgentConfBase
|
|
50
|
+
_agent_state: AgentStateBase
|
|
51
|
+
|
|
52
|
+
# prefix for all saved agent files
|
|
53
|
+
_saved_agent_suffix = ".pkl"
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def init_agent_conf(cls, env, config):
|
|
57
|
+
"""
|
|
58
|
+
Initialize the agent configuration.
|
|
59
|
+
"""
|
|
60
|
+
raise NotImplementedError
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def init_agent_state(cls, env, agent_conf: AgentConfBase, rng) -> AgentStateBase:
|
|
64
|
+
""" Initializes and returns the agent state (network params, optimizer state, etc.). """
|
|
65
|
+
raise NotImplementedError
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def build_train_fn(cls, env, agent_conf: AgentConfBase, mh: MetricsHandler = None):
|
|
69
|
+
""" Returns the train function. Takes (rng, agent_state, traj) and runs one training chunk. """
|
|
70
|
+
return lambda rng_key, agent_state, traj: cls._train_fn(rng_key, env, agent_conf, agent_state, traj=traj, mh=mh)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def _train_fn(cls, rng, env,
|
|
74
|
+
agent_conf: AgentConfBase,
|
|
75
|
+
agent_state: AgentStateBase = None,
|
|
76
|
+
traj=None,
|
|
77
|
+
mh: MetricsHandler = None):
|
|
78
|
+
""" Main training algorithm of an RL algorithm. """
|
|
79
|
+
raise NotImplementedError
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def build_eval_fn(cls, env, agent_conf: AgentConfBase, mh: MetricsHandler = None):
|
|
83
|
+
""" Returns the eval function. Same signature as build_train_fn:
|
|
84
|
+
takes (rng, agent_state, traj) and runs an evaluation rollout,
|
|
85
|
+
returning a dict with metric summaries. Does not mutate the agent's
|
|
86
|
+
training state (any local run_stats updates during eval are
|
|
87
|
+
discarded with the eval scan carry). """
|
|
88
|
+
return lambda rng_key, agent_state, traj: cls._eval_fn(rng_key, env, agent_conf, agent_state, traj=traj, mh=mh)
|
|
89
|
+
|
|
90
|
+
@classmethod
|
|
91
|
+
def _eval_fn(cls, rng, env,
|
|
92
|
+
agent_conf: AgentConfBase,
|
|
93
|
+
agent_state: AgentStateBase,
|
|
94
|
+
traj=None,
|
|
95
|
+
mh: MetricsHandler = None):
|
|
96
|
+
""" Evaluation rollout. Subclasses should implement this to run the
|
|
97
|
+
agent (greedy/deterministic if applicable) for a fixed number of
|
|
98
|
+
parallel env steps and return a metrics dict. """
|
|
99
|
+
raise NotImplementedError
|
|
100
|
+
|
|
101
|
+
@classmethod
|
|
102
|
+
def play_policy(cls, train_state, env, config, n_envs, n_steps=None, record=False, key=None):
|
|
103
|
+
raise NotImplementedError
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def save_agent(cls, path, agent_conf: AgentConfBase, agent_state: AgentStateBase):
|
|
107
|
+
""" Save the agent state to a file."""
|
|
108
|
+
path = Path(path)
|
|
109
|
+
path = path / (cls.__name__ + "_saved")
|
|
110
|
+
path = path.with_suffix(cls._saved_agent_suffix)
|
|
111
|
+
# serialize agent state
|
|
112
|
+
serialized_state = cls.serialize(agent_conf, agent_state)
|
|
113
|
+
# save agent state
|
|
114
|
+
with open(path, 'wb') as file:
|
|
115
|
+
pickle.dump(serialized_state, file)
|
|
116
|
+
print(f"\nSaved agent to: {path}\n")
|
|
117
|
+
return path
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def load_agent(cls, path):
|
|
121
|
+
""" Load the agent state from a file. """
|
|
122
|
+
if isinstance(path, str):
|
|
123
|
+
path = Path(path)
|
|
124
|
+
if not path.is_file():
|
|
125
|
+
raise ValueError(f'Not a file: {path}')
|
|
126
|
+
if path.suffix != cls._saved_agent_suffix:
|
|
127
|
+
raise ValueError(f'Not a {cls._saved_agent_suffix} file: {path}')
|
|
128
|
+
with open(path, 'rb') as file:
|
|
129
|
+
data = pickle.load(file)
|
|
130
|
+
return cls.from_dict(data)
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def serialize(cls, agent_conf: AgentConfBase, agent_state: AgentStateBase):
|
|
134
|
+
""" Serialize conf and state of an agent. """
|
|
135
|
+
serialized_agent_conf = agent_conf.serialize()
|
|
136
|
+
serialized_agent_state = agent_state.serialize()
|
|
137
|
+
return {"agent_conf": serialized_agent_conf,
|
|
138
|
+
"agent_state": serialized_agent_state}
|
|
139
|
+
|
|
140
|
+
@classmethod
|
|
141
|
+
def from_dict(cls, d):
|
|
142
|
+
""" Load conf and state of an agent from a dictionary. """
|
|
143
|
+
agent_conf = cls._agent_conf.from_dict(d["agent_conf"])
|
|
144
|
+
agent_state = cls._agent_state.from_dict(d["agent_state"], agent_conf)
|
|
145
|
+
return agent_conf, agent_state
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _wrap_env(env, config):
|
|
149
|
+
""" Wrap the environment with the necessary wrappers. """
|
|
150
|
+
raise NotImplementedError
|
|
151
|
+
|
|
152
|
+
@classmethod
|
|
153
|
+
def _linear_lr_schedule(cls, count, num_minibatches, update_epochs, lr, num_updates):
|
|
154
|
+
frac = (
|
|
155
|
+
1.0
|
|
156
|
+
- (count // (num_minibatches * update_epochs))
|
|
157
|
+
/ num_updates
|
|
158
|
+
)
|
|
159
|
+
return lr * frac
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
from typing import NamedTuple
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import jax
|
|
5
|
+
import jax.numpy as jnp
|
|
6
|
+
from flax import struct
|
|
7
|
+
from flax.training import train_state
|
|
8
|
+
|
|
9
|
+
from loco_mujoco.environments.base import TrajState
|
|
10
|
+
from loco_mujoco.core.wrappers.mjx import Metrics
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Transition(NamedTuple):
|
|
14
|
+
done: jnp.ndarray
|
|
15
|
+
absorbing: jnp.ndarray
|
|
16
|
+
action: jnp.ndarray
|
|
17
|
+
value: jnp.ndarray
|
|
18
|
+
reward: jnp.ndarray
|
|
19
|
+
log_prob: jnp.ndarray
|
|
20
|
+
obs: jnp.ndarray
|
|
21
|
+
info: jnp.ndarray
|
|
22
|
+
traj_state: TrajState
|
|
23
|
+
metrics: Metrics
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MetricHandlerTransition(NamedTuple):
|
|
27
|
+
env_state: Any
|
|
28
|
+
logged_metrics: Metrics
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TrainState(train_state.TrainState):
|
|
32
|
+
run_stats: Any
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@struct.dataclass
|
|
36
|
+
class TrainStateBuffer:
|
|
37
|
+
train_states: TrainState
|
|
38
|
+
n: int
|
|
39
|
+
size: int # buffer size
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def create(cls, train_state: TrainState, size: int):
|
|
43
|
+
return TrainStateBuffer(
|
|
44
|
+
train_states=jax.tree.map(lambda x: jnp.stack([x] * size), train_state),
|
|
45
|
+
n=0,
|
|
46
|
+
size=size
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def add(cls, train_state_buffer, train_state: TrainState):
|
|
51
|
+
index = train_state_buffer.n
|
|
52
|
+
# Add the new train state at index n
|
|
53
|
+
train_states_updated = jax.tree.map(
|
|
54
|
+
lambda buffer, new: buffer.at[index].set(new),
|
|
55
|
+
train_state_buffer.train_states,
|
|
56
|
+
train_state
|
|
57
|
+
)
|
|
58
|
+
return train_state_buffer.replace(
|
|
59
|
+
train_states=train_states_updated,
|
|
60
|
+
n=index + 1,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@struct.dataclass
|
|
65
|
+
class RewardNormStats:
|
|
66
|
+
"""
|
|
67
|
+
Running statistics for PPO-style reward normalization. Lives on the agent
|
|
68
|
+
state so it survives env resets and is saved with the agent.
|
|
69
|
+
"""
|
|
70
|
+
mean: jnp.ndarray
|
|
71
|
+
var: jnp.ndarray
|
|
72
|
+
count: jnp.ndarray
|
|
73
|
+
return_val: jnp.ndarray
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def create(cls, num_envs: int):
|
|
77
|
+
return cls(
|
|
78
|
+
mean=jnp.zeros(()),
|
|
79
|
+
var=jnp.ones(()),
|
|
80
|
+
count=jnp.full((), 1e-4),
|
|
81
|
+
return_val=jnp.zeros((num_envs,)),
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def update_reward_norm(stats: "RewardNormStats", reward: jnp.ndarray,
|
|
86
|
+
done: jnp.ndarray, gamma: float):
|
|
87
|
+
"""
|
|
88
|
+
Update running stats with a batch of rewards and return the normalized
|
|
89
|
+
reward plus the new stats. Mirrors the previous NormalizeVecReward wrapper.
|
|
90
|
+
"""
|
|
91
|
+
return_val = stats.return_val * gamma * (1.0 - done) + reward
|
|
92
|
+
|
|
93
|
+
batch_mean = jnp.mean(return_val, axis=0)
|
|
94
|
+
batch_var = jnp.var(return_val, axis=0)
|
|
95
|
+
batch_count = return_val.shape[0]
|
|
96
|
+
|
|
97
|
+
delta = batch_mean - stats.mean
|
|
98
|
+
tot_count = stats.count + batch_count
|
|
99
|
+
|
|
100
|
+
new_mean = stats.mean + delta * batch_count / tot_count
|
|
101
|
+
m_a = stats.var * stats.count
|
|
102
|
+
m_b = batch_var * batch_count
|
|
103
|
+
M2 = m_a + m_b + jnp.square(delta) * stats.count * batch_count / tot_count
|
|
104
|
+
new_var = M2 / tot_count
|
|
105
|
+
|
|
106
|
+
new_stats = RewardNormStats(
|
|
107
|
+
mean=new_mean,
|
|
108
|
+
var=new_var,
|
|
109
|
+
count=tot_count,
|
|
110
|
+
return_val=return_val,
|
|
111
|
+
)
|
|
112
|
+
normalized_reward = reward / jnp.sqrt(new_stats.var + 1e-8)
|
|
113
|
+
return normalized_reward, new_stats
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@struct.dataclass
|
|
117
|
+
class ReplayBuffer:
|
|
118
|
+
"""Simple circular replay buffer for off-policy / imitation algorithms.
|
|
119
|
+
|
|
120
|
+
Core columns: (obs, next_obs, action, reward, done). `value_target` is
|
|
121
|
+
an optional column used by value-distillation-style critic learning
|
|
122
|
+
(e.g. VanillaDagger stores `V_teacher(s)` there at collect time).
|
|
123
|
+
|
|
124
|
+
Any of `next_obs` / `value_target` can be turned off at `create` time;
|
|
125
|
+
disabled columns become zero-sized dummies (0 bytes). Flags are static
|
|
126
|
+
metadata so `if self.store_*` branches are baked at trace time.
|
|
127
|
+
|
|
128
|
+
Optional Bernoulli sub-sampling at write time: with `add_subsample=True`
|
|
129
|
+
and `add_prob=p`, each transition in an incoming batch is kept with
|
|
130
|
+
probability p. Effective temporal coverage becomes ~capacity / p without
|
|
131
|
+
growing GPU memory. `add_batch` requires an `rng` when subsampling is on.
|
|
132
|
+
"""
|
|
133
|
+
obs: jnp.ndarray
|
|
134
|
+
next_obs: jnp.ndarray
|
|
135
|
+
action: jnp.ndarray
|
|
136
|
+
reward: jnp.ndarray
|
|
137
|
+
done: jnp.ndarray
|
|
138
|
+
value_target: jnp.ndarray
|
|
139
|
+
ptr: int
|
|
140
|
+
size: int
|
|
141
|
+
store_next_obs: bool = struct.field(pytree_node=False, default=True)
|
|
142
|
+
store_value_target: bool = struct.field(pytree_node=False, default=False)
|
|
143
|
+
add_subsample: bool = struct.field(pytree_node=False, default=False)
|
|
144
|
+
add_prob: float = struct.field(pytree_node=False, default=1.0)
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def create(cls, obs_dim: int, action_dim: int, capacity: int,
|
|
148
|
+
store_next_obs: bool = True,
|
|
149
|
+
store_value_target: bool = False,
|
|
150
|
+
add_subsample: bool = False,
|
|
151
|
+
add_prob: float = 1.0):
|
|
152
|
+
next_obs_shape = (capacity, obs_dim) if store_next_obs else (0, 0)
|
|
153
|
+
value_shape = (capacity,) if store_value_target else (0,)
|
|
154
|
+
return cls(
|
|
155
|
+
obs=jnp.zeros((capacity, obs_dim)),
|
|
156
|
+
next_obs=jnp.zeros(next_obs_shape),
|
|
157
|
+
action=jnp.zeros((capacity, action_dim)),
|
|
158
|
+
reward=jnp.zeros((capacity,)),
|
|
159
|
+
done=jnp.zeros((capacity,), dtype=jnp.float32),
|
|
160
|
+
value_target=jnp.zeros(value_shape),
|
|
161
|
+
ptr=0,
|
|
162
|
+
size=0,
|
|
163
|
+
store_next_obs=store_next_obs,
|
|
164
|
+
store_value_target=store_value_target,
|
|
165
|
+
add_subsample=bool(add_subsample),
|
|
166
|
+
add_prob=float(add_prob),
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def add_batch(self, obs, next_obs, action, reward, done,
|
|
170
|
+
value_target=None, rng=None):
|
|
171
|
+
capacity = self.obs.shape[0]
|
|
172
|
+
batch_size = obs.shape[0]
|
|
173
|
+
|
|
174
|
+
if self.add_subsample:
|
|
175
|
+
assert rng is not None, (
|
|
176
|
+
"buffer was created with add_subsample=True but "
|
|
177
|
+
"add_batch received rng=None"
|
|
178
|
+
)
|
|
179
|
+
# Per-element Bernoulli keep mask.
|
|
180
|
+
keep = jax.random.uniform(rng, (batch_size,)) < self.add_prob
|
|
181
|
+
kept_cnt = jnp.sum(keep.astype(jnp.int32))
|
|
182
|
+
# Kept entries are written to consecutive slots; dropped entries get the
|
|
183
|
+
# out-of-range index `capacity`, which mode="drop" discards. They must not be
|
|
184
|
+
# routed to a real slot: a scatter with duplicate indices applies its updates
|
|
185
|
+
# in an unspecified order, so a dropped entry could overwrite a kept one.
|
|
186
|
+
offsets = jnp.cumsum(keep.astype(jnp.int32)) - 1
|
|
187
|
+
indices = jnp.where(keep, (self.ptr + offsets) % capacity, capacity)
|
|
188
|
+
if self.store_next_obs:
|
|
189
|
+
new_next_obs = self.next_obs.at[indices].set(next_obs, mode="drop")
|
|
190
|
+
else:
|
|
191
|
+
new_next_obs = self.next_obs
|
|
192
|
+
if self.store_value_target:
|
|
193
|
+
assert value_target is not None, (
|
|
194
|
+
"buffer was created with store_value_target=True but "
|
|
195
|
+
"add_batch received value_target=None"
|
|
196
|
+
)
|
|
197
|
+
new_value_target = self.value_target.at[indices].set(value_target, mode="drop")
|
|
198
|
+
else:
|
|
199
|
+
new_value_target = self.value_target
|
|
200
|
+
return self.replace(
|
|
201
|
+
obs=self.obs.at[indices].set(obs, mode="drop"),
|
|
202
|
+
next_obs=new_next_obs,
|
|
203
|
+
action=self.action.at[indices].set(action, mode="drop"),
|
|
204
|
+
reward=self.reward.at[indices].set(reward, mode="drop"),
|
|
205
|
+
done=self.done.at[indices].set(done.astype(jnp.float32), mode="drop"),
|
|
206
|
+
value_target=new_value_target,
|
|
207
|
+
ptr=(self.ptr + kept_cnt) % capacity,
|
|
208
|
+
size=jnp.minimum(self.size + kept_cnt, capacity),
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
indices = (self.ptr + jnp.arange(batch_size)) % capacity
|
|
212
|
+
new_next_obs = (self.next_obs.at[indices].set(next_obs)
|
|
213
|
+
if self.store_next_obs else self.next_obs)
|
|
214
|
+
if self.store_value_target:
|
|
215
|
+
assert value_target is not None, (
|
|
216
|
+
"buffer was created with store_value_target=True but "
|
|
217
|
+
"add_batch received value_target=None"
|
|
218
|
+
)
|
|
219
|
+
new_value_target = self.value_target.at[indices].set(value_target)
|
|
220
|
+
else:
|
|
221
|
+
new_value_target = self.value_target
|
|
222
|
+
return self.replace(
|
|
223
|
+
obs=self.obs.at[indices].set(obs),
|
|
224
|
+
next_obs=new_next_obs,
|
|
225
|
+
action=self.action.at[indices].set(action),
|
|
226
|
+
reward=self.reward.at[indices].set(reward),
|
|
227
|
+
done=self.done.at[indices].set(done.astype(jnp.float32)),
|
|
228
|
+
value_target=new_value_target,
|
|
229
|
+
ptr=(self.ptr + batch_size) % capacity,
|
|
230
|
+
size=jnp.minimum(self.size + batch_size, capacity),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def sample(self, rng, batch_size: int):
|
|
234
|
+
indices = jax.random.randint(rng, (batch_size,), 0, self.size)
|
|
235
|
+
if self.store_next_obs:
|
|
236
|
+
next_obs = self.next_obs[indices]
|
|
237
|
+
else:
|
|
238
|
+
next_obs = jnp.zeros((batch_size, 0), dtype=self.obs.dtype)
|
|
239
|
+
return (
|
|
240
|
+
self.obs[indices],
|
|
241
|
+
next_obs,
|
|
242
|
+
self.action[indices],
|
|
243
|
+
self.reward[indices],
|
|
244
|
+
self.done[indices],
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@struct.dataclass
|
|
249
|
+
class ReservoirBuffer:
|
|
250
|
+
"""Reservoir-sampled long-term memory.
|
|
251
|
+
|
|
252
|
+
Approximates a uniform sample of all transitions ever passed through
|
|
253
|
+
`add_batch`. Capacity stays fixed; early additions fill sequentially,
|
|
254
|
+
later additions probabilistically replace random slots. At steady
|
|
255
|
+
state each global transition has probability `capacity / total_seen`
|
|
256
|
+
of being in the buffer.
|
|
257
|
+
|
|
258
|
+
Columns mirror `ReplayBuffer` (obs / action / value_target). No
|
|
259
|
+
`next_obs` or `reward` / `done` — this is intended as a
|
|
260
|
+
long-term BC + value-distillation store, not a TD buffer.
|
|
261
|
+
|
|
262
|
+
`min_include_prob` optionally floors the per-item inclusion rate. The
|
|
263
|
+
pure reservoir's rate decays as k/n indefinitely, which is correct
|
|
264
|
+
for stationary streams but wastes signal when the data distribution
|
|
265
|
+
drifts (e.g. DAgger rollouts shift as the student learns). Setting
|
|
266
|
+
`min_include_prob > 0` caps the effective n at `ceil(k / p_min)`, so
|
|
267
|
+
the buffer behaves like a uniform sample over the last `k / p_min`
|
|
268
|
+
transitions once that window has filled. Default 0.0 = pure reservoir.
|
|
269
|
+
"""
|
|
270
|
+
obs: jnp.ndarray
|
|
271
|
+
action: jnp.ndarray
|
|
272
|
+
value_target: jnp.ndarray
|
|
273
|
+
total_seen: jnp.ndarray # scalar int32, counts all transitions ever added
|
|
274
|
+
size: jnp.ndarray # scalar int32, min(total_seen, capacity)
|
|
275
|
+
store_value_target: bool = struct.field(pytree_node=False, default=False)
|
|
276
|
+
min_include_prob: float = struct.field(pytree_node=False, default=0.0)
|
|
277
|
+
|
|
278
|
+
@classmethod
|
|
279
|
+
def create(cls, obs_dim: int, action_dim: int, capacity: int,
|
|
280
|
+
store_value_target: bool = False,
|
|
281
|
+
min_include_prob: float = 0.0):
|
|
282
|
+
value_shape = (capacity,) if store_value_target else (0,)
|
|
283
|
+
return cls(
|
|
284
|
+
obs=jnp.zeros((capacity, obs_dim)),
|
|
285
|
+
action=jnp.zeros((capacity, action_dim)),
|
|
286
|
+
value_target=jnp.zeros(value_shape),
|
|
287
|
+
total_seen=jnp.zeros((), dtype=jnp.int32),
|
|
288
|
+
size=jnp.zeros((), dtype=jnp.int32),
|
|
289
|
+
store_value_target=store_value_target,
|
|
290
|
+
min_include_prob=float(min_include_prob),
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
def add_batch(self, obs, action, value_target, rng):
|
|
294
|
+
"""Reservoir-insert a batch. Vectorised: for each element i with
|
|
295
|
+
global index g_i = total_seen + i, draw j_i ~ uniform_int[0, n_eff_i)
|
|
296
|
+
where `n_eff_i = min(g_i+1, ceil(capacity / min_include_prob))`.
|
|
297
|
+
Write at slot g_i if we're still in the fill phase, else write at
|
|
298
|
+
j_i when j_i < capacity. Collisions between two include=True
|
|
299
|
+
elements writing to the same slot are resolved by whichever wins
|
|
300
|
+
the scatter — a tiny bias but costs nothing.
|
|
301
|
+
"""
|
|
302
|
+
import math
|
|
303
|
+
capacity = self.obs.shape[0]
|
|
304
|
+
B = obs.shape[0]
|
|
305
|
+
batch_idx = self.total_seen + jnp.arange(B, dtype=jnp.int32)
|
|
306
|
+
|
|
307
|
+
# Effective denominator for the reservoir decision. A positive
|
|
308
|
+
# min_include_prob caps it at ceil(k/p_min), flooring the
|
|
309
|
+
# inclusion probability at p_min. Python-level branch — static.
|
|
310
|
+
if self.min_include_prob > 0.0:
|
|
311
|
+
cap_n = int(math.ceil(capacity / self.min_include_prob))
|
|
312
|
+
n_eff = jnp.minimum(batch_idx + 1, jnp.int32(cap_n))
|
|
313
|
+
else:
|
|
314
|
+
n_eff = batch_idx + 1
|
|
315
|
+
|
|
316
|
+
u = jax.random.uniform(rng, (B,))
|
|
317
|
+
j = jnp.floor(u * n_eff.astype(jnp.float32)).astype(jnp.int32)
|
|
318
|
+
|
|
319
|
+
fill = batch_idx < capacity
|
|
320
|
+
include = fill | (j < capacity)
|
|
321
|
+
slot = jnp.where(fill, batch_idx, j)
|
|
322
|
+
slot = jnp.clip(slot, 0, capacity - 1)
|
|
323
|
+
|
|
324
|
+
# Masked scatter: for non-include entries, write back the current
|
|
325
|
+
# slot contents (no-op). Collisions between include=True entries
|
|
326
|
+
# are rare for capacity >> batch_size.
|
|
327
|
+
fresh_obs = jnp.where(include[:, None], obs, self.obs[slot])
|
|
328
|
+
fresh_action = jnp.where(include[:, None], action, self.action[slot])
|
|
329
|
+
new_obs = self.obs.at[slot].set(fresh_obs)
|
|
330
|
+
new_action = self.action.at[slot].set(fresh_action)
|
|
331
|
+
|
|
332
|
+
if self.store_value_target:
|
|
333
|
+
assert value_target is not None, (
|
|
334
|
+
"reservoir was created with store_value_target=True but "
|
|
335
|
+
"add_batch received value_target=None"
|
|
336
|
+
)
|
|
337
|
+
fresh_vt = jnp.where(include, value_target, self.value_target[slot])
|
|
338
|
+
new_vt = self.value_target.at[slot].set(fresh_vt)
|
|
339
|
+
else:
|
|
340
|
+
new_vt = self.value_target
|
|
341
|
+
|
|
342
|
+
new_total = self.total_seen + B
|
|
343
|
+
new_size = jnp.minimum(jnp.int32(capacity), new_total)
|
|
344
|
+
return self.replace(
|
|
345
|
+
obs=new_obs,
|
|
346
|
+
action=new_action,
|
|
347
|
+
value_target=new_vt,
|
|
348
|
+
total_seen=new_total,
|
|
349
|
+
size=new_size,
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
def sample(self, rng, batch_size: int):
|
|
353
|
+
indices = jax.random.randint(rng, (batch_size,), 0, jnp.maximum(1, self.size))
|
|
354
|
+
return (
|
|
355
|
+
self.obs[indices],
|
|
356
|
+
self.action[indices],
|
|
357
|
+
(self.value_target[indices] if self.store_value_target
|
|
358
|
+
else jnp.zeros((batch_size,), dtype=self.obs.dtype)),
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
@struct.dataclass
|
|
363
|
+
class BestTrainStates:
|
|
364
|
+
train_states: TrainState
|
|
365
|
+
metrics: jnp.array
|
|
366
|
+
iterations: jnp.array
|
|
367
|
+
cur_worst_perf: float
|
|
368
|
+
step: int
|
|
369
|
+
n: int
|
|
370
|
+
size: int
|
|
371
|
+
|
|
372
|
+
@classmethod
|
|
373
|
+
def create(cls, train_state: TrainState, n: int):
|
|
374
|
+
return BestTrainStates(
|
|
375
|
+
train_states=jax.tree.map(lambda x: jnp.stack([x] * n), train_state),
|
|
376
|
+
metrics=jnp.full((n,), -jnp.inf),
|
|
377
|
+
iterations=jnp.zeros((n,)),
|
|
378
|
+
cur_worst_perf=-jnp.inf,
|
|
379
|
+
n=n,
|
|
380
|
+
size=0
|
|
381
|
+
)
|