robotenv 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. robotenv/__init__.py +21 -0
  2. robotenv/_canon.py +136 -0
  3. robotenv/config/__init__.py +93 -0
  4. robotenv/config/base_env.py +557 -0
  5. robotenv/config/control.py +161 -0
  6. robotenv/config/defaults.py +29 -0
  7. robotenv/config/env.py +107 -0
  8. robotenv/config/kinematics.py +121 -0
  9. robotenv/config/mount.py +172 -0
  10. robotenv/config/providers.py +171 -0
  11. robotenv/config/robot_providers.py +140 -0
  12. robotenv/config/sim.py +152 -0
  13. robotenv/config/specs.py +111 -0
  14. robotenv/config/sweep.py +100 -0
  15. robotenv/config/task.py +183 -0
  16. robotenv/envs/__init__.py +34 -0
  17. robotenv/envs/spaces.py +38 -0
  18. robotenv/errors.py +35 -0
  19. robotenv/robot/__init__.py +30 -0
  20. robotenv/robot/arm.py +319 -0
  21. robotenv/robot/external.py +234 -0
  22. robotenv/robot/generators.py +138 -0
  23. robotenv/robot/gripper.py +82 -0
  24. robotenv/robot/pose.py +49 -0
  25. robotenv/robot/providers/__init__.py +5 -0
  26. robotenv/robot/providers/menagerie/__init__.py +127 -0
  27. robotenv/robot/providers/menagerie/_describe.py +130 -0
  28. robotenv/robot/providers/menagerie/cards.json +37 -0
  29. robotenv/simulators/__init__.py +51 -0
  30. robotenv/simulators/base.py +70 -0
  31. robotenv/simulators/mujoco/__init__.py +38 -0
  32. robotenv/simulators/mujoco/arm_spec.py +137 -0
  33. robotenv/simulators/mujoco/compile.py +242 -0
  34. robotenv/simulators/mujoco/env.py +257 -0
  35. robotenv/simulators/mujoco/external.py +443 -0
  36. robotenv/simulators/mujoco/mount.py +65 -0
  37. robotenv/simulators/mujoco/render.py +158 -0
  38. robotenv/simulators/mujoco/scenes.py +35 -0
  39. robotenv/simulators/mujoco/sim.py +174 -0
  40. robotenv/simulators/warp/__init__.py +100 -0
  41. robotenv/simulators/warp/env.py +235 -0
  42. robotenv/simulators/warp/sim.py +332 -0
  43. robotenv/tasks/__init__.py +10 -0
  44. robotenv/tasks/base.py +147 -0
  45. robotenv/tasks/empty/__init__.py +42 -0
  46. robotenv/tasks/empty/params.py +33 -0
  47. robotenv/tasks/empty/task.py +27 -0
  48. robotenv/tasks/robosuite/__init__.py +133 -0
  49. robotenv/tasks/robosuite/_compat.py +165 -0
  50. robotenv/tasks/robosuite/_introspect.py +65 -0
  51. robotenv/tasks/robosuite/params.py +152 -0
  52. robotenv/tasks/robosuite/scene.py +140 -0
  53. robotenv/tasks/robosuite/task.py +145 -0
  54. robotenv/viz/__init__.py +22 -0
  55. robotenv/viz/_deps.py +16 -0
  56. robotenv/viz/camera.py +98 -0
  57. robotenv/viz/clip.py +180 -0
  58. robotenv/viz/frames.py +93 -0
  59. robotenv/viz/record.py +141 -0
  60. robotenv-0.2.1.dist-info/METADATA +136 -0
  61. robotenv-0.2.1.dist-info/RECORD +63 -0
  62. robotenv-0.2.1.dist-info/WHEEL +4 -0
  63. robotenv-0.2.1.dist-info/licenses/LICENSE +339 -0
@@ -0,0 +1,145 @@
1
+ """Engine side of ``robosuite/<EnvName>``: robosuite's scene in our MuJoCo simulation, our objective on top."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+ import torch
7
+ from torchrl.data import Composite, Unbounded
8
+
9
+ from ..base import BaseTask
10
+ from .params import LiftObject, ReachObject
11
+ from .scene import MOUNT_SITE, PREFIX, load_scene
12
+
13
+
14
+ def _reseed_generators(sampler, seed: int) -> None:
15
+ """robosuite >= 1.5.2 samples from a numpy Generator the sampler holds; reseed it in place."""
16
+ rng = getattr(sampler, "rng", None)
17
+ if rng is not None and hasattr(rng, "bit_generator"):
18
+ rng.bit_generator.state = np.random.default_rng(seed).bit_generator.state
19
+ for sub in getattr(sampler, "samplers", {}).values():
20
+ _reseed_generators(sub, seed)
21
+
22
+
23
+ class RobosuiteTask(BaseTask):
24
+ """Observations (the ``task`` group), all in the robot's base frame like every other task:
25
+
26
+ ``ee_pos`` (3), and for every physical object of the scene ``<name>_pos`` (3) and ``<name>_quat`` (4, w x y z);
27
+ visual-only objects are in the scene but not observed. Reward is ``-||ee - object||`` for the objective's
28
+ object; success is the objective's test.
29
+
30
+ Objects with a free joint are placed by robosuite's sampler at every reset. Objects without one (a door frame,
31
+ a stand, a target bin) are fixtures: all envs share one model, so they are placed once, when the task is bound,
32
+ from ``sim.seed``. Objects the sampler does not place stay at their pose in the model.
33
+ """
34
+
35
+ def __init__(self, cfg):
36
+ self.cfg = cfg
37
+ params = cfg.task.params
38
+ self._objective = params.objective
39
+ self._assets = load_scene(params.name, params.kwargs, cfg.sim.seed)
40
+ self._items: list[tuple[str, int, int | None]] = [] # observed objects: (name, body id, free-joint qpos address)
41
+ self._by_name: dict[str, tuple[str, int, int | None]] = {}
42
+ self.fixtures: tuple[str, ...] = () # objects placed once at bind time (no free joint)
43
+
44
+ @property
45
+ def object_names(self) -> tuple[str, ...]:
46
+ """Names of the observed (physical) objects."""
47
+ return tuple(o.name for o in self._assets.objects if not o.visual)
48
+
49
+ # --- model -----------------------------------------------------------------------
50
+
51
+ def attach(self, spec) -> None:
52
+ import mujoco
53
+
54
+ child = mujoco.MjSpec.from_string(self._assets.xml)
55
+ spec.attach(child, frame=spec.worldbody.add_frame(), prefix=PREFIX)
56
+
57
+ def mount_site(self) -> str:
58
+ return PREFIX + MOUNT_SITE # the stock robot's root pose, where the scene mounts a robot
59
+
60
+ def bind(self, sim) -> None:
61
+ import mujoco
62
+
63
+ model = sim.model
64
+ self._items, every = [], {}
65
+ for o in self._assets.objects:
66
+ body = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, PREFIX + o.root_body)
67
+ adr = None
68
+ if o.joints:
69
+ jid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, PREFIX + o.joints[0])
70
+ if jid >= 0 and model.jnt_type[jid] == mujoco.mjtJoint.mjJNT_FREE:
71
+ adr = int(model.jnt_qposadr[jid])
72
+ every[o.name] = (body, adr)
73
+ if not o.visual:
74
+ self._items.append((o.name, body, adr))
75
+ self._by_name = {it[0]: it for it in self._items}
76
+ # Fixtures cannot be placed per env (they live in the shared model): one placement, baked in.
77
+ fixtures = []
78
+ for name, (pos, quat, _obj) in self._sample(int(self.cfg.sim.seed)).items(): # same seed as the scene
79
+ body, adr = every.get(name, (-1, None))
80
+ if body >= 0 and adr is None:
81
+ model.body_pos[body] = np.asarray(pos, dtype=float)
82
+ model.body_quat[body] = np.asarray(quat, dtype=float)
83
+ fixtures.append(name)
84
+ self.fixtures = tuple(fixtures)
85
+
86
+ def observation_spec(self, num_envs: int):
87
+ f32 = torch.float32
88
+ entries = {"ee_pos": Unbounded(shape=(num_envs, 3), dtype=f32)}
89
+ for name, _, _ in self._items:
90
+ entries[f"{name}_pos"] = Unbounded(shape=(num_envs, 3), dtype=f32)
91
+ entries[f"{name}_quat"] = Unbounded(shape=(num_envs, 4), dtype=f32)
92
+ return Composite(**entries, shape=(num_envs,))
93
+
94
+ # --- reset -----------------------------------------------------------------------
95
+
96
+ def _sample(self, seed: int) -> dict:
97
+ sampler = self._assets.sampler
98
+ if sampler is None:
99
+ return {}
100
+ state = np.random.get_state()
101
+ np.random.seed(seed % 2**32) # robosuite 1.5.1 samples from the global stream; put it back afterwards
102
+ try:
103
+ _reseed_generators(sampler, seed)
104
+ return sampler.sample()
105
+ finally:
106
+ np.random.set_state(state)
107
+
108
+ def reset(self, sim, mask, rand, injected) -> None:
109
+ draws = (rand(sim.num_envs) * 2**31).to(torch.int64).numpy() # one per env, whatever the mask
110
+ values: dict[str, torch.Tensor] = {}
111
+ placed: dict[str, torch.Tensor] = {}
112
+ for i in torch.nonzero(mask.cpu()).flatten().tolist():
113
+ for name, (pos, quat, _obj) in self._sample(int(draws[i])).items():
114
+ item = self._by_name.get(name)
115
+ if item is not None and item[2] is not None:
116
+ if name not in values:
117
+ values[name] = torch.zeros(sim.num_envs, 7, dtype=torch.float64)
118
+ placed[name] = torch.zeros(sim.num_envs, dtype=torch.bool)
119
+ values[name][i] = torch.from_numpy(np.concatenate([np.asarray(pos), np.asarray(quat)]))
120
+ placed[name][i] = True
121
+ for name, v in values.items():
122
+ sim.write_qpos(self._by_name[name][2], v, placed[name])
123
+
124
+ # --- observations, success, reward -----------------------------------------------
125
+
126
+ def observe(self, sim) -> dict[str, torch.Tensor]:
127
+ out = {"ee_pos": sim.site_pos(sim.iface.ee_site_id, "base")}
128
+ for name, body, _ in self._items:
129
+ out[f"{name}_pos"], out[f"{name}_quat"] = sim.body_pose(body, "base")
130
+ return out
131
+
132
+ def _dist(self, obs) -> torch.Tensor:
133
+ return torch.linalg.norm(obs["ee_pos"] - obs[f"{self._objective.object}_pos"], dim=-1, keepdim=True)
134
+
135
+ def success(self, sim, obs) -> torch.Tensor:
136
+ obj = self._objective
137
+ if isinstance(obj, ReachObject):
138
+ return self._dist(obs) < obj.tolerance
139
+ assert isinstance(obj, LiftObject)
140
+ body = self._by_name[obj.object][1]
141
+ z = sim.body_pose(body, "world")[0][:, 2:3]
142
+ return z >= obj.height
143
+
144
+ def reward(self, sim, obs) -> torch.Tensor:
145
+ return -self._dist(obs)
@@ -0,0 +1,22 @@
1
+ """Visualization: ``Clip`` (save GIF / MP4, show inline), frame recording, camera framing.
2
+
3
+ Importing this package imports numpy only. Pillow, imageio-ffmpeg and mujoco are loaded when first needed.
4
+ """
5
+
6
+ from .camera import VIEWS, CameraSpec, auto_camera, workspace_points
7
+ from .clip import Clip
8
+ from .frames import annotate, tile, tile_clips
9
+ from .record import FrameRecorder, record_rollout
10
+
11
+ __all__ = [
12
+ "VIEWS",
13
+ "CameraSpec",
14
+ "Clip",
15
+ "FrameRecorder",
16
+ "annotate",
17
+ "auto_camera",
18
+ "record_rollout",
19
+ "tile",
20
+ "tile_clips",
21
+ "workspace_points",
22
+ ]
robotenv/viz/_deps.py ADDED
@@ -0,0 +1,16 @@
1
+ """Optional-dependency helpers. No third-party import at module level."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib.util
6
+
7
+ INSTALL_HINT = "pip install 'robotenv[viz]'"
8
+
9
+
10
+ def have(module: str) -> bool:
11
+ return importlib.util.find_spec(module) is not None
12
+
13
+
14
+ def need(module: str, what: str) -> None:
15
+ if not have(module):
16
+ raise ImportError(f"{what} needs the optional dependency {module!r}; install it with {INSTALL_HINT}")
robotenv/viz/camera.py ADDED
@@ -0,0 +1,98 @@
1
+ """Camera framing from configs alone: numpy only, no engine import until ``CameraSpec.to_mjv``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from dataclasses import dataclass
7
+
8
+ import numpy as np
9
+
10
+ from ..config import FixedBox, world_mount
11
+ from ..config import kinematics as kin
12
+
13
+ # (azimuth, elevation) in degrees, world frame. Azimuth 0 looks along +x; the camera sits opposite to its view
14
+ # direction, so "front" (180) looks at the robot from +x towards -x.
15
+ VIEWS: dict[str, tuple[float, float]] = {
16
+ "iso": (135.0, -25.0),
17
+ "front": (180.0, -15.0),
18
+ "side": (90.0, -10.0),
19
+ "top": (90.0, -89.0),
20
+ }
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class CameraSpec:
25
+ """A free camera: look-at point, distance, azimuth and elevation (degrees), as in MuJoCo."""
26
+
27
+ lookat: tuple[float, float, float]
28
+ distance: float
29
+ azimuth: float
30
+ elevation: float
31
+
32
+ def to_mjv(self):
33
+ import mujoco
34
+
35
+ cam = mujoco.MjvCamera()
36
+ cam.type = mujoco.mjtCamera.mjCAMERA_FREE
37
+ cam.lookat[:] = self.lookat
38
+ cam.distance = self.distance
39
+ cam.azimuth = self.azimuth
40
+ cam.elevation = self.elevation
41
+ return cam
42
+
43
+
44
+ def _parts(cfg):
45
+ """``cfg`` is an ``EnvConfig`` (robot, mount and target region) or a bare ``ParametricArm`` (at the origin)."""
46
+ robot = getattr(cfg, "robot", cfg)
47
+ task = getattr(cfg, "task", None)
48
+ return robot, getattr(task, "target_region", None), None if task is None else world_mount(cfg)
49
+
50
+
51
+ def workspace_points(cfg, n_samples: int = 512, seed: int = 0) -> np.ndarray:
52
+ """World-frame points the camera should cover: base, floor under the base, end-effector samples over the
53
+ joint ranges, the home pose, and the corners of a ``FixedBox`` target region. Uses its own random
54
+ generator, so it never touches an environment's random stream."""
55
+ robot, region, mount = _parts(cfg)
56
+ rng = np.random.default_rng(seed)
57
+ T = np.eye(4) if mount is None else kin.pose_to_matrix(mount)
58
+ base = T[:3, 3]
59
+ pts = [base[None], np.array([[base[0], base[1], 0.0]])]
60
+ if getattr(robot, "end_effector", True) is None:
61
+ # A robot without an end effector (a quadruped) has no reach to sample: frame a metre around where it starts.
62
+ root = getattr(robot, "home_root", None)
63
+ centre = base if root is None else T[:3, :3] @ np.asarray(root.pos) + base
64
+ pts.append(centre + 0.6 * np.array([[x, y, z] for x in (-1, 1) for y in (-1, 1) for z in (-1, 1)], dtype=float))
65
+ else:
66
+ pts.append(kin.forward_ee_batch(robot, kin.sample_joint_positions(robot, n_samples, rng), mount))
67
+ pts.append(kin.forward_ee_batch(robot, np.asarray(robot.home_qpos, dtype=float)[None], mount))
68
+ if isinstance(region, FixedBox):
69
+ lo, hi = np.asarray(region.low, dtype=float), np.asarray(region.high, dtype=float)
70
+ corners = np.array([[x, y, z] for x in (lo[0], hi[0]) for y in (lo[1], hi[1]) for z in (lo[2], hi[2])])
71
+ pts.append(corners @ T[:3, :3].T + T[:3, 3])
72
+ return np.concatenate(pts, axis=0)
73
+
74
+
75
+ def auto_camera(
76
+ *cfgs,
77
+ view: str = "iso",
78
+ fovy: float = 45.0,
79
+ aspect: float = 4.0 / 3.0,
80
+ margin: float = 1.15,
81
+ n_samples: int = 512,
82
+ ) -> CameraSpec:
83
+ """A camera that sees the whole workspace of every config in ``cfgs``.
84
+
85
+ Giving several configs returns ONE camera that covers all of them, so clips of different embodiments are
86
+ drawn at the same scale. ``fovy`` (degrees) and ``aspect`` (width / height) are those of the render.
87
+ """
88
+ if not cfgs:
89
+ raise ValueError("auto_camera needs at least one config")
90
+ if view not in VIEWS:
91
+ raise ValueError(f"unknown view {view!r}; available: {sorted(VIEWS)}")
92
+ pts = np.concatenate([workspace_points(c, n_samples) for c in cfgs], axis=0)
93
+ center = (pts.min(axis=0) + pts.max(axis=0)) / 2.0
94
+ radius = max(float(np.linalg.norm(pts - center, axis=1).max()), 0.1)
95
+ # The tighter of the vertical and horizontal half angles decides how far back the camera must sit.
96
+ half = math.atan(math.tan(math.radians(fovy) / 2.0) * min(1.0, aspect))
97
+ az, el = VIEWS[view]
98
+ return CameraSpec(tuple(float(x) for x in center), margin * radius / math.sin(half), az, el)
robotenv/viz/clip.py ADDED
@@ -0,0 +1,180 @@
1
+ """``Clip``: a stack of RGB frames with a frame rate. Saves GIF or MP4 and displays inline in notebooks."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import html
7
+ import io
8
+ import tempfile
9
+ import warnings
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+
15
+ from ._deps import have, need
16
+
17
+ FORMATS = ("mp4", "gif")
18
+ _GIF_MIN_MS = 20 # most browsers play shorter GIF delays at 100 ms
19
+ _EMBED_WARN_MB = 25.0
20
+
21
+
22
+ def pad_even(frames: np.ndarray) -> np.ndarray:
23
+ """Repeat the last row / column so height and width are even (yuv420p needs even sizes)."""
24
+ h, w = frames.shape[1:3]
25
+ if h % 2 == 0 and w % 2 == 0:
26
+ return frames
27
+ return np.pad(frames, ((0, 0), (0, h % 2), (0, w % 2), (0, 0)), mode="edge")
28
+
29
+
30
+ def gif_delay_ms(fps: float) -> int:
31
+ """Frame delay stored in a GIF. GIF counts in 10 ms steps, so only some rates are exact."""
32
+ wanted = 1000.0 / fps
33
+ delay = max(int(round(wanted / 10.0)) * 10, _GIF_MIN_MS)
34
+ if abs(delay - wanted) > 0.5:
35
+ warnings.warn(
36
+ f"GIF stores frame delays in 10 ms steps (minimum {_GIF_MIN_MS} ms): {fps:g} fps will play at "
37
+ f"{1000.0 / delay:.4g} fps. Use 50, 25, 20 or 10 fps for an exact rate, or save an mp4.",
38
+ UserWarning,
39
+ stacklevel=3,
40
+ )
41
+ return delay
42
+
43
+
44
+ def _encode_gif(frames: np.ndarray, fps: float) -> bytes:
45
+ need("PIL", "GIF output")
46
+ from PIL import Image
47
+
48
+ delay = gif_delay_ms(fps)
49
+ images = [Image.fromarray(f) for f in frames]
50
+ buf = io.BytesIO()
51
+ images[0].save(buf, format="GIF", save_all=True, append_images=images[1:], duration=delay, loop=0)
52
+ return buf.getvalue()
53
+
54
+
55
+ def _write_mp4(frames: np.ndarray, fps: float, path: Path) -> None:
56
+ need("imageio", "MP4 output")
57
+ need("imageio_ffmpeg", "MP4 output")
58
+ import imageio.v2 as iio
59
+
60
+ iio.mimsave(
61
+ path,
62
+ list(pad_even(frames)),
63
+ fps=fps,
64
+ codec="libx264",
65
+ pixelformat="yuv420p",
66
+ macro_block_size=1, # sizes are already even; the default would rescale to a multiple of 16
67
+ output_params=["-crf", "23", "-preset", "medium", "-movflags", "+faststart"],
68
+ )
69
+
70
+
71
+ @dataclass(frozen=True, eq=False, repr=False)
72
+ class Clip:
73
+ """``frames`` is ``uint8 (T, H, W, 3)``; ``fps`` is the playback rate. Treat both as immutable."""
74
+
75
+ frames: np.ndarray
76
+ fps: float
77
+
78
+ def __post_init__(self):
79
+ f = np.asarray(self.frames)
80
+ if f.dtype != np.uint8 or f.ndim != 4 or f.shape[-1] != 3:
81
+ raise ValueError(f"frames must be uint8 with shape (T, H, W, 3), got {f.dtype} {f.shape}")
82
+ if min(f.shape[:3]) < 1:
83
+ raise ValueError(f"a clip needs at least one non-empty frame, got shape {f.shape}")
84
+ fps = float(self.fps)
85
+ if not (fps > 0.0 and np.isfinite(fps)):
86
+ raise ValueError(f"fps must be a positive number, got {self.fps!r}")
87
+ object.__setattr__(self, "frames", np.ascontiguousarray(f))
88
+ object.__setattr__(self, "fps", fps)
89
+
90
+ def __len__(self) -> int:
91
+ return int(self.frames.shape[0])
92
+
93
+ @property
94
+ def size(self) -> tuple[int, int]:
95
+ """``(height, width)`` of one frame."""
96
+ return int(self.frames.shape[1]), int(self.frames.shape[2])
97
+
98
+ @property
99
+ def duration(self) -> float:
100
+ """Playing time in seconds."""
101
+ return len(self) / self.fps
102
+
103
+ def __repr__(self) -> str:
104
+ h, w = self.size
105
+ return f"Clip({len(self)} frames, {h}x{w}, {self.fps:g} fps, {self.duration:.2f} s)"
106
+
107
+ def with_fps(self, fps: float) -> "Clip":
108
+ return Clip(self.frames, fps)
109
+
110
+ # --- files -------------------------------------------------------------------------------------
111
+
112
+ def to_bytes(self, fmt: str = "mp4") -> bytes:
113
+ fmt = _check_format(fmt)
114
+ if fmt == "gif":
115
+ return _encode_gif(self.frames, self.fps)
116
+ with tempfile.TemporaryDirectory() as d:
117
+ path = Path(d) / "clip.mp4"
118
+ _write_mp4(self.frames, self.fps, path)
119
+ return path.read_bytes()
120
+
121
+ def save(self, path, fmt: str | None = None) -> Path:
122
+ """Write the clip. The format is ``fmt`` or the suffix of ``path`` (``.mp4`` or ``.gif``)."""
123
+ path = Path(path)
124
+ fmt = _check_format(fmt or path.suffix.lstrip(".").lower(), what=f"{path.name!r}")
125
+ path.parent.mkdir(parents=True, exist_ok=True)
126
+ if fmt == "gif":
127
+ path.write_bytes(_encode_gif(self.frames, self.fps))
128
+ else:
129
+ _write_mp4(self.frames, self.fps, path)
130
+ return path
131
+
132
+ # --- notebooks ---------------------------------------------------------------------------------
133
+
134
+ def _html(self, fmt: str | None) -> str:
135
+ note = ""
136
+ if fmt is None: # automatic: MP4 when it can be written, else GIF
137
+ if have("imageio") and have("imageio_ffmpeg"):
138
+ fmt = "mp4"
139
+ else:
140
+ need("PIL", "inline display (neither MP4 nor GIF can be written)")
141
+ fmt = "gif"
142
+ note = "MP4 needs imageio-ffmpeg (pip install 'robotenv[viz]'); showing a GIF instead."
143
+ data = self.to_bytes(fmt)
144
+ if len(data) / 1e6 > _EMBED_WARN_MB:
145
+ warnings.warn(
146
+ f"embedding a {len(data) / 1e6:.0f} MB {fmt} in the notebook will make it slow; "
147
+ "save it and open the file instead, or record fewer frames (every=...)",
148
+ UserWarning,
149
+ stacklevel=3,
150
+ )
151
+ b64 = base64.b64encode(data).decode("ascii")
152
+ if fmt == "mp4":
153
+ media = (
154
+ '<video controls loop autoplay muted playsinline style="max-width:100%" '
155
+ f'src="data:video/mp4;base64,{b64}"></video>'
156
+ )
157
+ else:
158
+ media = f'<img style="max-width:100%" src="data:image/gif;base64,{b64}"/>'
159
+ if note:
160
+ media += f'<div style="font:12px sans-serif;color:#666">{html.escape(note)}</div>'
161
+ return media
162
+
163
+ def _repr_html_(self) -> str:
164
+ """Shown by Jupyter / VS Code when a ``Clip`` is the last expression of a cell."""
165
+ try:
166
+ return self._html(None)
167
+ except ImportError as e:
168
+ return f"<pre>{html.escape(repr(self))}\n{html.escape(str(e))}</pre>"
169
+
170
+ def show(self, fmt: str | None = None) -> None:
171
+ """Display inline now. ``fmt`` is ``"mp4"`` or ``"gif"``; ``None`` picks MP4 when it is available."""
172
+ from IPython.display import HTML, display
173
+
174
+ display(HTML(self._html(None if fmt is None else _check_format(fmt))))
175
+
176
+
177
+ def _check_format(fmt: str, what: str = "format") -> str:
178
+ if fmt not in FORMATS:
179
+ raise ValueError(f"cannot write {what}: expected one of {list(FORMATS)}, got {fmt!r}")
180
+ return fmt
robotenv/viz/frames.py ADDED
@@ -0,0 +1,93 @@
1
+ """Frame helpers: text labels and tiling. Pillow is imported only by ``annotate``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from typing import Sequence
7
+
8
+ import numpy as np
9
+
10
+ from ._deps import need
11
+ from .clip import Clip
12
+
13
+
14
+ def _font(size: int):
15
+ from PIL import ImageFont
16
+
17
+ try:
18
+ return ImageFont.load_default(size=size)
19
+ except TypeError: # Pillow < 10.1: fixed-size bitmap font
20
+ return ImageFont.load_default()
21
+
22
+
23
+ def annotate(frame: np.ndarray, text: str, *, corner: str = "tl", size: int = 14) -> np.ndarray:
24
+ """Draw white text on a translucent dark box in a corner (``tl``, ``tr``, ``bl``, ``br``)."""
25
+ need("PIL", "annotate()")
26
+ from PIL import Image, ImageDraw
27
+
28
+ if corner not in ("tl", "tr", "bl", "br"):
29
+ raise ValueError(f"corner must be one of tl, tr, bl, br; got {corner!r}")
30
+ base = Image.fromarray(np.asarray(frame)).convert("RGBA")
31
+ layer = Image.new("RGBA", base.size, (0, 0, 0, 0))
32
+ draw = ImageDraw.Draw(layer)
33
+ font = _font(size)
34
+ left, top, right, bottom = draw.textbbox((0, 0), text, font=font)
35
+ pad = 3
36
+ bw, bh = right - left + 2 * pad, bottom - top + 2 * pad
37
+ x = 0 if corner in ("tl", "bl") else base.width - bw
38
+ y = 0 if corner in ("tl", "tr") else base.height - bh
39
+ draw.rectangle([x, y, x + bw, y + bh], fill=(0, 0, 0, 170))
40
+ draw.text((x + pad - left, y + pad - top), text, font=font, fill=(255, 255, 255, 255))
41
+ return np.asarray(Image.alpha_composite(base, layer).convert("RGB"))
42
+
43
+
44
+ def _grid(n: int, ncols: int | None) -> tuple[int, int]:
45
+ if n < 1:
46
+ raise ValueError("nothing to tile")
47
+ if ncols is None:
48
+ ncols = math.ceil(math.sqrt(n))
49
+ if ncols < 1:
50
+ raise ValueError(f"ncols must be >= 1, got {ncols}")
51
+ ncols = min(int(ncols), n)
52
+ return math.ceil(n / ncols), ncols
53
+
54
+
55
+ def _tile_array(arr: np.ndarray, ncols: int | None, pad: int) -> np.ndarray:
56
+ """``(N, T, H, W, 3) -> (T, rows*H + ..., cols*W + ..., 3)``; a border of ``pad`` pixels around every tile."""
57
+ n, t, h, w, _ = arr.shape
58
+ rows, cols = _grid(n, ncols)
59
+ out = np.zeros((t, rows * h + (rows + 1) * pad, cols * w + (cols + 1) * pad, 3), dtype=np.uint8)
60
+ for k in range(n):
61
+ r, c = divmod(k, cols)
62
+ y, x = pad + r * (h + pad), pad + c * (w + pad)
63
+ out[:, y : y + h, x : x + w] = arr[k]
64
+ return out
65
+
66
+
67
+ def tile(frames: Sequence[np.ndarray], ncols: int | None = None, pad: int = 2) -> np.ndarray:
68
+ """Arrange equally sized ``(H, W, 3)`` frames in a grid (near-square unless ``ncols`` is given)."""
69
+ arr = np.stack([np.asarray(f) for f in frames])
70
+ if arr.dtype != np.uint8 or arr.ndim != 4 or arr.shape[-1] != 3:
71
+ raise ValueError(f"frames must be equally sized uint8 (H, W, 3) images, got {arr.dtype} {arr.shape}")
72
+ return _tile_array(arr[:, None], ncols, pad)[0]
73
+
74
+
75
+ def tile_clips(clips: Sequence[Clip], ncols: int | None = None, labels: Sequence[str] | None = None, pad: int = 2) -> Clip:
76
+ """Play clips side by side. They need the same frame size and fps; the result is as long as the shortest."""
77
+ clips = list(clips)
78
+ if not clips:
79
+ raise ValueError("nothing to tile")
80
+ if len({c.size for c in clips}) != 1:
81
+ raise ValueError(f"clips must have the same frame size, got {[c.size for c in clips]}")
82
+ if len({c.fps for c in clips}) != 1:
83
+ raise ValueError(f"clips must have the same fps, got {[c.fps for c in clips]}")
84
+ if labels is not None and len(labels) != len(clips):
85
+ raise ValueError(f"expected {len(clips)} labels, got {len(labels)}")
86
+ t = min(len(c) for c in clips)
87
+ stacks = []
88
+ for k, c in enumerate(clips):
89
+ f = c.frames[:t]
90
+ if labels is not None:
91
+ f = np.stack([annotate(x, labels[k]) for x in f])
92
+ stacks.append(f)
93
+ return Clip(_tile_array(np.stack(stacks), ncols, pad), clips[0].fps)