robotenv 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robotenv/__init__.py +21 -0
- robotenv/_canon.py +136 -0
- robotenv/config/__init__.py +93 -0
- robotenv/config/base_env.py +557 -0
- robotenv/config/control.py +161 -0
- robotenv/config/defaults.py +29 -0
- robotenv/config/env.py +107 -0
- robotenv/config/kinematics.py +121 -0
- robotenv/config/mount.py +172 -0
- robotenv/config/providers.py +171 -0
- robotenv/config/robot_providers.py +140 -0
- robotenv/config/sim.py +152 -0
- robotenv/config/specs.py +111 -0
- robotenv/config/sweep.py +100 -0
- robotenv/config/task.py +183 -0
- robotenv/envs/__init__.py +34 -0
- robotenv/envs/spaces.py +38 -0
- robotenv/errors.py +35 -0
- robotenv/robot/__init__.py +30 -0
- robotenv/robot/arm.py +319 -0
- robotenv/robot/external.py +234 -0
- robotenv/robot/generators.py +138 -0
- robotenv/robot/gripper.py +82 -0
- robotenv/robot/pose.py +49 -0
- robotenv/robot/providers/__init__.py +5 -0
- robotenv/robot/providers/menagerie/__init__.py +127 -0
- robotenv/robot/providers/menagerie/_describe.py +130 -0
- robotenv/robot/providers/menagerie/cards.json +37 -0
- robotenv/simulators/__init__.py +51 -0
- robotenv/simulators/base.py +70 -0
- robotenv/simulators/mujoco/__init__.py +38 -0
- robotenv/simulators/mujoco/arm_spec.py +137 -0
- robotenv/simulators/mujoco/compile.py +242 -0
- robotenv/simulators/mujoco/env.py +257 -0
- robotenv/simulators/mujoco/external.py +443 -0
- robotenv/simulators/mujoco/mount.py +65 -0
- robotenv/simulators/mujoco/render.py +158 -0
- robotenv/simulators/mujoco/scenes.py +35 -0
- robotenv/simulators/mujoco/sim.py +174 -0
- robotenv/simulators/warp/__init__.py +100 -0
- robotenv/simulators/warp/env.py +235 -0
- robotenv/simulators/warp/sim.py +332 -0
- robotenv/tasks/__init__.py +10 -0
- robotenv/tasks/base.py +147 -0
- robotenv/tasks/empty/__init__.py +42 -0
- robotenv/tasks/empty/params.py +33 -0
- robotenv/tasks/empty/task.py +27 -0
- robotenv/tasks/robosuite/__init__.py +133 -0
- robotenv/tasks/robosuite/_compat.py +165 -0
- robotenv/tasks/robosuite/_introspect.py +65 -0
- robotenv/tasks/robosuite/params.py +152 -0
- robotenv/tasks/robosuite/scene.py +140 -0
- robotenv/tasks/robosuite/task.py +145 -0
- robotenv/viz/__init__.py +22 -0
- robotenv/viz/_deps.py +16 -0
- robotenv/viz/camera.py +98 -0
- robotenv/viz/clip.py +180 -0
- robotenv/viz/frames.py +93 -0
- robotenv/viz/record.py +141 -0
- robotenv-0.2.1.dist-info/METADATA +136 -0
- robotenv-0.2.1.dist-info/RECORD +63 -0
- robotenv-0.2.1.dist-info/WHEEL +4 -0
- robotenv-0.2.1.dist-info/licenses/LICENSE +339 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Engine side of ``robosuite/<EnvName>``: robosuite's scene in our MuJoCo simulation, our objective on top."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
from torchrl.data import Composite, Unbounded
|
|
8
|
+
|
|
9
|
+
from ..base import BaseTask
|
|
10
|
+
from .params import LiftObject, ReachObject
|
|
11
|
+
from .scene import MOUNT_SITE, PREFIX, load_scene
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _reseed_generators(sampler, seed: int) -> None:
|
|
15
|
+
"""robosuite >= 1.5.2 samples from a numpy Generator the sampler holds; reseed it in place."""
|
|
16
|
+
rng = getattr(sampler, "rng", None)
|
|
17
|
+
if rng is not None and hasattr(rng, "bit_generator"):
|
|
18
|
+
rng.bit_generator.state = np.random.default_rng(seed).bit_generator.state
|
|
19
|
+
for sub in getattr(sampler, "samplers", {}).values():
|
|
20
|
+
_reseed_generators(sub, seed)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RobosuiteTask(BaseTask):
|
|
24
|
+
"""Observations (the ``task`` group), all in the robot's base frame like every other task:
|
|
25
|
+
|
|
26
|
+
``ee_pos`` (3), and for every physical object of the scene ``<name>_pos`` (3) and ``<name>_quat`` (4, w x y z);
|
|
27
|
+
visual-only objects are in the scene but not observed. Reward is ``-||ee - object||`` for the objective's
|
|
28
|
+
object; success is the objective's test.
|
|
29
|
+
|
|
30
|
+
Objects with a free joint are placed by robosuite's sampler at every reset. Objects without one (a door frame,
|
|
31
|
+
a stand, a target bin) are fixtures: all envs share one model, so they are placed once, when the task is bound,
|
|
32
|
+
from ``sim.seed``. Objects the sampler does not place stay at their pose in the model.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, cfg):
|
|
36
|
+
self.cfg = cfg
|
|
37
|
+
params = cfg.task.params
|
|
38
|
+
self._objective = params.objective
|
|
39
|
+
self._assets = load_scene(params.name, params.kwargs, cfg.sim.seed)
|
|
40
|
+
self._items: list[tuple[str, int, int | None]] = [] # observed objects: (name, body id, free-joint qpos address)
|
|
41
|
+
self._by_name: dict[str, tuple[str, int, int | None]] = {}
|
|
42
|
+
self.fixtures: tuple[str, ...] = () # objects placed once at bind time (no free joint)
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def object_names(self) -> tuple[str, ...]:
|
|
46
|
+
"""Names of the observed (physical) objects."""
|
|
47
|
+
return tuple(o.name for o in self._assets.objects if not o.visual)
|
|
48
|
+
|
|
49
|
+
# --- model -----------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
def attach(self, spec) -> None:
|
|
52
|
+
import mujoco
|
|
53
|
+
|
|
54
|
+
child = mujoco.MjSpec.from_string(self._assets.xml)
|
|
55
|
+
spec.attach(child, frame=spec.worldbody.add_frame(), prefix=PREFIX)
|
|
56
|
+
|
|
57
|
+
def mount_site(self) -> str:
|
|
58
|
+
return PREFIX + MOUNT_SITE # the stock robot's root pose, where the scene mounts a robot
|
|
59
|
+
|
|
60
|
+
def bind(self, sim) -> None:
|
|
61
|
+
import mujoco
|
|
62
|
+
|
|
63
|
+
model = sim.model
|
|
64
|
+
self._items, every = [], {}
|
|
65
|
+
for o in self._assets.objects:
|
|
66
|
+
body = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, PREFIX + o.root_body)
|
|
67
|
+
adr = None
|
|
68
|
+
if o.joints:
|
|
69
|
+
jid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, PREFIX + o.joints[0])
|
|
70
|
+
if jid >= 0 and model.jnt_type[jid] == mujoco.mjtJoint.mjJNT_FREE:
|
|
71
|
+
adr = int(model.jnt_qposadr[jid])
|
|
72
|
+
every[o.name] = (body, adr)
|
|
73
|
+
if not o.visual:
|
|
74
|
+
self._items.append((o.name, body, adr))
|
|
75
|
+
self._by_name = {it[0]: it for it in self._items}
|
|
76
|
+
# Fixtures cannot be placed per env (they live in the shared model): one placement, baked in.
|
|
77
|
+
fixtures = []
|
|
78
|
+
for name, (pos, quat, _obj) in self._sample(int(self.cfg.sim.seed)).items(): # same seed as the scene
|
|
79
|
+
body, adr = every.get(name, (-1, None))
|
|
80
|
+
if body >= 0 and adr is None:
|
|
81
|
+
model.body_pos[body] = np.asarray(pos, dtype=float)
|
|
82
|
+
model.body_quat[body] = np.asarray(quat, dtype=float)
|
|
83
|
+
fixtures.append(name)
|
|
84
|
+
self.fixtures = tuple(fixtures)
|
|
85
|
+
|
|
86
|
+
def observation_spec(self, num_envs: int):
|
|
87
|
+
f32 = torch.float32
|
|
88
|
+
entries = {"ee_pos": Unbounded(shape=(num_envs, 3), dtype=f32)}
|
|
89
|
+
for name, _, _ in self._items:
|
|
90
|
+
entries[f"{name}_pos"] = Unbounded(shape=(num_envs, 3), dtype=f32)
|
|
91
|
+
entries[f"{name}_quat"] = Unbounded(shape=(num_envs, 4), dtype=f32)
|
|
92
|
+
return Composite(**entries, shape=(num_envs,))
|
|
93
|
+
|
|
94
|
+
# --- reset -----------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
def _sample(self, seed: int) -> dict:
|
|
97
|
+
sampler = self._assets.sampler
|
|
98
|
+
if sampler is None:
|
|
99
|
+
return {}
|
|
100
|
+
state = np.random.get_state()
|
|
101
|
+
np.random.seed(seed % 2**32) # robosuite 1.5.1 samples from the global stream; put it back afterwards
|
|
102
|
+
try:
|
|
103
|
+
_reseed_generators(sampler, seed)
|
|
104
|
+
return sampler.sample()
|
|
105
|
+
finally:
|
|
106
|
+
np.random.set_state(state)
|
|
107
|
+
|
|
108
|
+
def reset(self, sim, mask, rand, injected) -> None:
|
|
109
|
+
draws = (rand(sim.num_envs) * 2**31).to(torch.int64).numpy() # one per env, whatever the mask
|
|
110
|
+
values: dict[str, torch.Tensor] = {}
|
|
111
|
+
placed: dict[str, torch.Tensor] = {}
|
|
112
|
+
for i in torch.nonzero(mask.cpu()).flatten().tolist():
|
|
113
|
+
for name, (pos, quat, _obj) in self._sample(int(draws[i])).items():
|
|
114
|
+
item = self._by_name.get(name)
|
|
115
|
+
if item is not None and item[2] is not None:
|
|
116
|
+
if name not in values:
|
|
117
|
+
values[name] = torch.zeros(sim.num_envs, 7, dtype=torch.float64)
|
|
118
|
+
placed[name] = torch.zeros(sim.num_envs, dtype=torch.bool)
|
|
119
|
+
values[name][i] = torch.from_numpy(np.concatenate([np.asarray(pos), np.asarray(quat)]))
|
|
120
|
+
placed[name][i] = True
|
|
121
|
+
for name, v in values.items():
|
|
122
|
+
sim.write_qpos(self._by_name[name][2], v, placed[name])
|
|
123
|
+
|
|
124
|
+
# --- observations, success, reward -----------------------------------------------
|
|
125
|
+
|
|
126
|
+
def observe(self, sim) -> dict[str, torch.Tensor]:
|
|
127
|
+
out = {"ee_pos": sim.site_pos(sim.iface.ee_site_id, "base")}
|
|
128
|
+
for name, body, _ in self._items:
|
|
129
|
+
out[f"{name}_pos"], out[f"{name}_quat"] = sim.body_pose(body, "base")
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
def _dist(self, obs) -> torch.Tensor:
|
|
133
|
+
return torch.linalg.norm(obs["ee_pos"] - obs[f"{self._objective.object}_pos"], dim=-1, keepdim=True)
|
|
134
|
+
|
|
135
|
+
def success(self, sim, obs) -> torch.Tensor:
|
|
136
|
+
obj = self._objective
|
|
137
|
+
if isinstance(obj, ReachObject):
|
|
138
|
+
return self._dist(obs) < obj.tolerance
|
|
139
|
+
assert isinstance(obj, LiftObject)
|
|
140
|
+
body = self._by_name[obj.object][1]
|
|
141
|
+
z = sim.body_pose(body, "world")[0][:, 2:3]
|
|
142
|
+
return z >= obj.height
|
|
143
|
+
|
|
144
|
+
def reward(self, sim, obs) -> torch.Tensor:
|
|
145
|
+
return -self._dist(obs)
|
robotenv/viz/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Visualization: ``Clip`` (save GIF / MP4, show inline), frame recording, camera framing.
|
|
2
|
+
|
|
3
|
+
Importing this package imports numpy only. Pillow, imageio-ffmpeg and mujoco are loaded when first needed.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .camera import VIEWS, CameraSpec, auto_camera, workspace_points
|
|
7
|
+
from .clip import Clip
|
|
8
|
+
from .frames import annotate, tile, tile_clips
|
|
9
|
+
from .record import FrameRecorder, record_rollout
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"VIEWS",
|
|
13
|
+
"CameraSpec",
|
|
14
|
+
"Clip",
|
|
15
|
+
"FrameRecorder",
|
|
16
|
+
"annotate",
|
|
17
|
+
"auto_camera",
|
|
18
|
+
"record_rollout",
|
|
19
|
+
"tile",
|
|
20
|
+
"tile_clips",
|
|
21
|
+
"workspace_points",
|
|
22
|
+
]
|
robotenv/viz/_deps.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Optional-dependency helpers. No third-party import at module level."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
|
|
7
|
+
INSTALL_HINT = "pip install 'robotenv[viz]'"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def have(module: str) -> bool:
|
|
11
|
+
return importlib.util.find_spec(module) is not None
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def need(module: str, what: str) -> None:
|
|
15
|
+
if not have(module):
|
|
16
|
+
raise ImportError(f"{what} needs the optional dependency {module!r}; install it with {INSTALL_HINT}")
|
robotenv/viz/camera.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Camera framing from configs alone: numpy only, no engine import until ``CameraSpec.to_mjv``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from ..config import FixedBox, world_mount
|
|
11
|
+
from ..config import kinematics as kin
|
|
12
|
+
|
|
13
|
+
# (azimuth, elevation) in degrees, world frame. Azimuth 0 looks along +x; the camera sits opposite to its view
|
|
14
|
+
# direction, so "front" (180) looks at the robot from +x towards -x.
|
|
15
|
+
VIEWS: dict[str, tuple[float, float]] = {
|
|
16
|
+
"iso": (135.0, -25.0),
|
|
17
|
+
"front": (180.0, -15.0),
|
|
18
|
+
"side": (90.0, -10.0),
|
|
19
|
+
"top": (90.0, -89.0),
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class CameraSpec:
|
|
25
|
+
"""A free camera: look-at point, distance, azimuth and elevation (degrees), as in MuJoCo."""
|
|
26
|
+
|
|
27
|
+
lookat: tuple[float, float, float]
|
|
28
|
+
distance: float
|
|
29
|
+
azimuth: float
|
|
30
|
+
elevation: float
|
|
31
|
+
|
|
32
|
+
def to_mjv(self):
|
|
33
|
+
import mujoco
|
|
34
|
+
|
|
35
|
+
cam = mujoco.MjvCamera()
|
|
36
|
+
cam.type = mujoco.mjtCamera.mjCAMERA_FREE
|
|
37
|
+
cam.lookat[:] = self.lookat
|
|
38
|
+
cam.distance = self.distance
|
|
39
|
+
cam.azimuth = self.azimuth
|
|
40
|
+
cam.elevation = self.elevation
|
|
41
|
+
return cam
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _parts(cfg):
|
|
45
|
+
"""``cfg`` is an ``EnvConfig`` (robot, mount and target region) or a bare ``ParametricArm`` (at the origin)."""
|
|
46
|
+
robot = getattr(cfg, "robot", cfg)
|
|
47
|
+
task = getattr(cfg, "task", None)
|
|
48
|
+
return robot, getattr(task, "target_region", None), None if task is None else world_mount(cfg)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def workspace_points(cfg, n_samples: int = 512, seed: int = 0) -> np.ndarray:
|
|
52
|
+
"""World-frame points the camera should cover: base, floor under the base, end-effector samples over the
|
|
53
|
+
joint ranges, the home pose, and the corners of a ``FixedBox`` target region. Uses its own random
|
|
54
|
+
generator, so it never touches an environment's random stream."""
|
|
55
|
+
robot, region, mount = _parts(cfg)
|
|
56
|
+
rng = np.random.default_rng(seed)
|
|
57
|
+
T = np.eye(4) if mount is None else kin.pose_to_matrix(mount)
|
|
58
|
+
base = T[:3, 3]
|
|
59
|
+
pts = [base[None], np.array([[base[0], base[1], 0.0]])]
|
|
60
|
+
if getattr(robot, "end_effector", True) is None:
|
|
61
|
+
# A robot without an end effector (a quadruped) has no reach to sample: frame a metre around where it starts.
|
|
62
|
+
root = getattr(robot, "home_root", None)
|
|
63
|
+
centre = base if root is None else T[:3, :3] @ np.asarray(root.pos) + base
|
|
64
|
+
pts.append(centre + 0.6 * np.array([[x, y, z] for x in (-1, 1) for y in (-1, 1) for z in (-1, 1)], dtype=float))
|
|
65
|
+
else:
|
|
66
|
+
pts.append(kin.forward_ee_batch(robot, kin.sample_joint_positions(robot, n_samples, rng), mount))
|
|
67
|
+
pts.append(kin.forward_ee_batch(robot, np.asarray(robot.home_qpos, dtype=float)[None], mount))
|
|
68
|
+
if isinstance(region, FixedBox):
|
|
69
|
+
lo, hi = np.asarray(region.low, dtype=float), np.asarray(region.high, dtype=float)
|
|
70
|
+
corners = np.array([[x, y, z] for x in (lo[0], hi[0]) for y in (lo[1], hi[1]) for z in (lo[2], hi[2])])
|
|
71
|
+
pts.append(corners @ T[:3, :3].T + T[:3, 3])
|
|
72
|
+
return np.concatenate(pts, axis=0)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def auto_camera(
|
|
76
|
+
*cfgs,
|
|
77
|
+
view: str = "iso",
|
|
78
|
+
fovy: float = 45.0,
|
|
79
|
+
aspect: float = 4.0 / 3.0,
|
|
80
|
+
margin: float = 1.15,
|
|
81
|
+
n_samples: int = 512,
|
|
82
|
+
) -> CameraSpec:
|
|
83
|
+
"""A camera that sees the whole workspace of every config in ``cfgs``.
|
|
84
|
+
|
|
85
|
+
Giving several configs returns ONE camera that covers all of them, so clips of different embodiments are
|
|
86
|
+
drawn at the same scale. ``fovy`` (degrees) and ``aspect`` (width / height) are those of the render.
|
|
87
|
+
"""
|
|
88
|
+
if not cfgs:
|
|
89
|
+
raise ValueError("auto_camera needs at least one config")
|
|
90
|
+
if view not in VIEWS:
|
|
91
|
+
raise ValueError(f"unknown view {view!r}; available: {sorted(VIEWS)}")
|
|
92
|
+
pts = np.concatenate([workspace_points(c, n_samples) for c in cfgs], axis=0)
|
|
93
|
+
center = (pts.min(axis=0) + pts.max(axis=0)) / 2.0
|
|
94
|
+
radius = max(float(np.linalg.norm(pts - center, axis=1).max()), 0.1)
|
|
95
|
+
# The tighter of the vertical and horizontal half angles decides how far back the camera must sit.
|
|
96
|
+
half = math.atan(math.tan(math.radians(fovy) / 2.0) * min(1.0, aspect))
|
|
97
|
+
az, el = VIEWS[view]
|
|
98
|
+
return CameraSpec(tuple(float(x) for x in center), margin * radius / math.sin(half), az, el)
|
robotenv/viz/clip.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""``Clip``: a stack of RGB frames with a frame rate. Saves GIF or MP4 and displays inline in notebooks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import html
|
|
7
|
+
import io
|
|
8
|
+
import tempfile
|
|
9
|
+
import warnings
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from ._deps import have, need
|
|
16
|
+
|
|
17
|
+
FORMATS = ("mp4", "gif")
|
|
18
|
+
_GIF_MIN_MS = 20 # most browsers play shorter GIF delays at 100 ms
|
|
19
|
+
_EMBED_WARN_MB = 25.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def pad_even(frames: np.ndarray) -> np.ndarray:
|
|
23
|
+
"""Repeat the last row / column so height and width are even (yuv420p needs even sizes)."""
|
|
24
|
+
h, w = frames.shape[1:3]
|
|
25
|
+
if h % 2 == 0 and w % 2 == 0:
|
|
26
|
+
return frames
|
|
27
|
+
return np.pad(frames, ((0, 0), (0, h % 2), (0, w % 2), (0, 0)), mode="edge")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def gif_delay_ms(fps: float) -> int:
|
|
31
|
+
"""Frame delay stored in a GIF. GIF counts in 10 ms steps, so only some rates are exact."""
|
|
32
|
+
wanted = 1000.0 / fps
|
|
33
|
+
delay = max(int(round(wanted / 10.0)) * 10, _GIF_MIN_MS)
|
|
34
|
+
if abs(delay - wanted) > 0.5:
|
|
35
|
+
warnings.warn(
|
|
36
|
+
f"GIF stores frame delays in 10 ms steps (minimum {_GIF_MIN_MS} ms): {fps:g} fps will play at "
|
|
37
|
+
f"{1000.0 / delay:.4g} fps. Use 50, 25, 20 or 10 fps for an exact rate, or save an mp4.",
|
|
38
|
+
UserWarning,
|
|
39
|
+
stacklevel=3,
|
|
40
|
+
)
|
|
41
|
+
return delay
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _encode_gif(frames: np.ndarray, fps: float) -> bytes:
|
|
45
|
+
need("PIL", "GIF output")
|
|
46
|
+
from PIL import Image
|
|
47
|
+
|
|
48
|
+
delay = gif_delay_ms(fps)
|
|
49
|
+
images = [Image.fromarray(f) for f in frames]
|
|
50
|
+
buf = io.BytesIO()
|
|
51
|
+
images[0].save(buf, format="GIF", save_all=True, append_images=images[1:], duration=delay, loop=0)
|
|
52
|
+
return buf.getvalue()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _write_mp4(frames: np.ndarray, fps: float, path: Path) -> None:
|
|
56
|
+
need("imageio", "MP4 output")
|
|
57
|
+
need("imageio_ffmpeg", "MP4 output")
|
|
58
|
+
import imageio.v2 as iio
|
|
59
|
+
|
|
60
|
+
iio.mimsave(
|
|
61
|
+
path,
|
|
62
|
+
list(pad_even(frames)),
|
|
63
|
+
fps=fps,
|
|
64
|
+
codec="libx264",
|
|
65
|
+
pixelformat="yuv420p",
|
|
66
|
+
macro_block_size=1, # sizes are already even; the default would rescale to a multiple of 16
|
|
67
|
+
output_params=["-crf", "23", "-preset", "medium", "-movflags", "+faststart"],
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True, eq=False, repr=False)
|
|
72
|
+
class Clip:
|
|
73
|
+
"""``frames`` is ``uint8 (T, H, W, 3)``; ``fps`` is the playback rate. Treat both as immutable."""
|
|
74
|
+
|
|
75
|
+
frames: np.ndarray
|
|
76
|
+
fps: float
|
|
77
|
+
|
|
78
|
+
def __post_init__(self):
|
|
79
|
+
f = np.asarray(self.frames)
|
|
80
|
+
if f.dtype != np.uint8 or f.ndim != 4 or f.shape[-1] != 3:
|
|
81
|
+
raise ValueError(f"frames must be uint8 with shape (T, H, W, 3), got {f.dtype} {f.shape}")
|
|
82
|
+
if min(f.shape[:3]) < 1:
|
|
83
|
+
raise ValueError(f"a clip needs at least one non-empty frame, got shape {f.shape}")
|
|
84
|
+
fps = float(self.fps)
|
|
85
|
+
if not (fps > 0.0 and np.isfinite(fps)):
|
|
86
|
+
raise ValueError(f"fps must be a positive number, got {self.fps!r}")
|
|
87
|
+
object.__setattr__(self, "frames", np.ascontiguousarray(f))
|
|
88
|
+
object.__setattr__(self, "fps", fps)
|
|
89
|
+
|
|
90
|
+
def __len__(self) -> int:
|
|
91
|
+
return int(self.frames.shape[0])
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def size(self) -> tuple[int, int]:
|
|
95
|
+
"""``(height, width)`` of one frame."""
|
|
96
|
+
return int(self.frames.shape[1]), int(self.frames.shape[2])
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def duration(self) -> float:
|
|
100
|
+
"""Playing time in seconds."""
|
|
101
|
+
return len(self) / self.fps
|
|
102
|
+
|
|
103
|
+
def __repr__(self) -> str:
|
|
104
|
+
h, w = self.size
|
|
105
|
+
return f"Clip({len(self)} frames, {h}x{w}, {self.fps:g} fps, {self.duration:.2f} s)"
|
|
106
|
+
|
|
107
|
+
def with_fps(self, fps: float) -> "Clip":
|
|
108
|
+
return Clip(self.frames, fps)
|
|
109
|
+
|
|
110
|
+
# --- files -------------------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
def to_bytes(self, fmt: str = "mp4") -> bytes:
|
|
113
|
+
fmt = _check_format(fmt)
|
|
114
|
+
if fmt == "gif":
|
|
115
|
+
return _encode_gif(self.frames, self.fps)
|
|
116
|
+
with tempfile.TemporaryDirectory() as d:
|
|
117
|
+
path = Path(d) / "clip.mp4"
|
|
118
|
+
_write_mp4(self.frames, self.fps, path)
|
|
119
|
+
return path.read_bytes()
|
|
120
|
+
|
|
121
|
+
def save(self, path, fmt: str | None = None) -> Path:
|
|
122
|
+
"""Write the clip. The format is ``fmt`` or the suffix of ``path`` (``.mp4`` or ``.gif``)."""
|
|
123
|
+
path = Path(path)
|
|
124
|
+
fmt = _check_format(fmt or path.suffix.lstrip(".").lower(), what=f"{path.name!r}")
|
|
125
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
if fmt == "gif":
|
|
127
|
+
path.write_bytes(_encode_gif(self.frames, self.fps))
|
|
128
|
+
else:
|
|
129
|
+
_write_mp4(self.frames, self.fps, path)
|
|
130
|
+
return path
|
|
131
|
+
|
|
132
|
+
# --- notebooks ---------------------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
def _html(self, fmt: str | None) -> str:
|
|
135
|
+
note = ""
|
|
136
|
+
if fmt is None: # automatic: MP4 when it can be written, else GIF
|
|
137
|
+
if have("imageio") and have("imageio_ffmpeg"):
|
|
138
|
+
fmt = "mp4"
|
|
139
|
+
else:
|
|
140
|
+
need("PIL", "inline display (neither MP4 nor GIF can be written)")
|
|
141
|
+
fmt = "gif"
|
|
142
|
+
note = "MP4 needs imageio-ffmpeg (pip install 'robotenv[viz]'); showing a GIF instead."
|
|
143
|
+
data = self.to_bytes(fmt)
|
|
144
|
+
if len(data) / 1e6 > _EMBED_WARN_MB:
|
|
145
|
+
warnings.warn(
|
|
146
|
+
f"embedding a {len(data) / 1e6:.0f} MB {fmt} in the notebook will make it slow; "
|
|
147
|
+
"save it and open the file instead, or record fewer frames (every=...)",
|
|
148
|
+
UserWarning,
|
|
149
|
+
stacklevel=3,
|
|
150
|
+
)
|
|
151
|
+
b64 = base64.b64encode(data).decode("ascii")
|
|
152
|
+
if fmt == "mp4":
|
|
153
|
+
media = (
|
|
154
|
+
'<video controls loop autoplay muted playsinline style="max-width:100%" '
|
|
155
|
+
f'src="data:video/mp4;base64,{b64}"></video>'
|
|
156
|
+
)
|
|
157
|
+
else:
|
|
158
|
+
media = f'<img style="max-width:100%" src="data:image/gif;base64,{b64}"/>'
|
|
159
|
+
if note:
|
|
160
|
+
media += f'<div style="font:12px sans-serif;color:#666">{html.escape(note)}</div>'
|
|
161
|
+
return media
|
|
162
|
+
|
|
163
|
+
def _repr_html_(self) -> str:
|
|
164
|
+
"""Shown by Jupyter / VS Code when a ``Clip`` is the last expression of a cell."""
|
|
165
|
+
try:
|
|
166
|
+
return self._html(None)
|
|
167
|
+
except ImportError as e:
|
|
168
|
+
return f"<pre>{html.escape(repr(self))}\n{html.escape(str(e))}</pre>"
|
|
169
|
+
|
|
170
|
+
def show(self, fmt: str | None = None) -> None:
|
|
171
|
+
"""Display inline now. ``fmt`` is ``"mp4"`` or ``"gif"``; ``None`` picks MP4 when it is available."""
|
|
172
|
+
from IPython.display import HTML, display
|
|
173
|
+
|
|
174
|
+
display(HTML(self._html(None if fmt is None else _check_format(fmt))))
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _check_format(fmt: str, what: str = "format") -> str:
|
|
178
|
+
if fmt not in FORMATS:
|
|
179
|
+
raise ValueError(f"cannot write {what}: expected one of {list(FORMATS)}, got {fmt!r}")
|
|
180
|
+
return fmt
|
robotenv/viz/frames.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Frame helpers: text labels and tiling. Pillow is imported only by ``annotate``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from typing import Sequence
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from ._deps import need
|
|
11
|
+
from .clip import Clip
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _font(size: int):
|
|
15
|
+
from PIL import ImageFont
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
return ImageFont.load_default(size=size)
|
|
19
|
+
except TypeError: # Pillow < 10.1: fixed-size bitmap font
|
|
20
|
+
return ImageFont.load_default()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def annotate(frame: np.ndarray, text: str, *, corner: str = "tl", size: int = 14) -> np.ndarray:
|
|
24
|
+
"""Draw white text on a translucent dark box in a corner (``tl``, ``tr``, ``bl``, ``br``)."""
|
|
25
|
+
need("PIL", "annotate()")
|
|
26
|
+
from PIL import Image, ImageDraw
|
|
27
|
+
|
|
28
|
+
if corner not in ("tl", "tr", "bl", "br"):
|
|
29
|
+
raise ValueError(f"corner must be one of tl, tr, bl, br; got {corner!r}")
|
|
30
|
+
base = Image.fromarray(np.asarray(frame)).convert("RGBA")
|
|
31
|
+
layer = Image.new("RGBA", base.size, (0, 0, 0, 0))
|
|
32
|
+
draw = ImageDraw.Draw(layer)
|
|
33
|
+
font = _font(size)
|
|
34
|
+
left, top, right, bottom = draw.textbbox((0, 0), text, font=font)
|
|
35
|
+
pad = 3
|
|
36
|
+
bw, bh = right - left + 2 * pad, bottom - top + 2 * pad
|
|
37
|
+
x = 0 if corner in ("tl", "bl") else base.width - bw
|
|
38
|
+
y = 0 if corner in ("tl", "tr") else base.height - bh
|
|
39
|
+
draw.rectangle([x, y, x + bw, y + bh], fill=(0, 0, 0, 170))
|
|
40
|
+
draw.text((x + pad - left, y + pad - top), text, font=font, fill=(255, 255, 255, 255))
|
|
41
|
+
return np.asarray(Image.alpha_composite(base, layer).convert("RGB"))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _grid(n: int, ncols: int | None) -> tuple[int, int]:
|
|
45
|
+
if n < 1:
|
|
46
|
+
raise ValueError("nothing to tile")
|
|
47
|
+
if ncols is None:
|
|
48
|
+
ncols = math.ceil(math.sqrt(n))
|
|
49
|
+
if ncols < 1:
|
|
50
|
+
raise ValueError(f"ncols must be >= 1, got {ncols}")
|
|
51
|
+
ncols = min(int(ncols), n)
|
|
52
|
+
return math.ceil(n / ncols), ncols
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _tile_array(arr: np.ndarray, ncols: int | None, pad: int) -> np.ndarray:
|
|
56
|
+
"""``(N, T, H, W, 3) -> (T, rows*H + ..., cols*W + ..., 3)``; a border of ``pad`` pixels around every tile."""
|
|
57
|
+
n, t, h, w, _ = arr.shape
|
|
58
|
+
rows, cols = _grid(n, ncols)
|
|
59
|
+
out = np.zeros((t, rows * h + (rows + 1) * pad, cols * w + (cols + 1) * pad, 3), dtype=np.uint8)
|
|
60
|
+
for k in range(n):
|
|
61
|
+
r, c = divmod(k, cols)
|
|
62
|
+
y, x = pad + r * (h + pad), pad + c * (w + pad)
|
|
63
|
+
out[:, y : y + h, x : x + w] = arr[k]
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def tile(frames: Sequence[np.ndarray], ncols: int | None = None, pad: int = 2) -> np.ndarray:
|
|
68
|
+
"""Arrange equally sized ``(H, W, 3)`` frames in a grid (near-square unless ``ncols`` is given)."""
|
|
69
|
+
arr = np.stack([np.asarray(f) for f in frames])
|
|
70
|
+
if arr.dtype != np.uint8 or arr.ndim != 4 or arr.shape[-1] != 3:
|
|
71
|
+
raise ValueError(f"frames must be equally sized uint8 (H, W, 3) images, got {arr.dtype} {arr.shape}")
|
|
72
|
+
return _tile_array(arr[:, None], ncols, pad)[0]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def tile_clips(clips: Sequence[Clip], ncols: int | None = None, labels: Sequence[str] | None = None, pad: int = 2) -> Clip:
|
|
76
|
+
"""Play clips side by side. They need the same frame size and fps; the result is as long as the shortest."""
|
|
77
|
+
clips = list(clips)
|
|
78
|
+
if not clips:
|
|
79
|
+
raise ValueError("nothing to tile")
|
|
80
|
+
if len({c.size for c in clips}) != 1:
|
|
81
|
+
raise ValueError(f"clips must have the same frame size, got {[c.size for c in clips]}")
|
|
82
|
+
if len({c.fps for c in clips}) != 1:
|
|
83
|
+
raise ValueError(f"clips must have the same fps, got {[c.fps for c in clips]}")
|
|
84
|
+
if labels is not None and len(labels) != len(clips):
|
|
85
|
+
raise ValueError(f"expected {len(clips)} labels, got {len(labels)}")
|
|
86
|
+
t = min(len(c) for c in clips)
|
|
87
|
+
stacks = []
|
|
88
|
+
for k, c in enumerate(clips):
|
|
89
|
+
f = c.frames[:t]
|
|
90
|
+
if labels is not None:
|
|
91
|
+
f = np.stack([annotate(x, labels[k]) for x in f])
|
|
92
|
+
stacks.append(f)
|
|
93
|
+
return Clip(_tile_array(np.stack(stacks), ncols, pad), clips[0].fps)
|