manifold-cli 0.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. manifold_cli/__init__.py +3 -0
  2. manifold_cli/__main__.py +4 -0
  3. manifold_cli/bridge.py +70 -0
  4. manifold_cli/cli.py +54 -0
  5. manifold_cli/clients/__init__.py +0 -0
  6. manifold_cli/clients/auth_flow.py +169 -0
  7. manifold_cli/clients/credentials.py +53 -0
  8. manifold_cli/clients/docker_engine.py +43 -0
  9. manifold_cli/clients/image.py +130 -0
  10. manifold_cli/clients/manifold.py +1288 -0
  11. manifold_cli/clients/runner_home.py +165 -0
  12. manifold_cli/clients/workos.py +99 -0
  13. manifold_cli/commands/__init__.py +0 -0
  14. manifold_cli/commands/auth.py +101 -0
  15. manifold_cli/commands/benchmark.py +539 -0
  16. manifold_cli/commands/organization.py +154 -0
  17. manifold_cli/commands/policy.py +749 -0
  18. manifold_cli/commands/run.py +337 -0
  19. manifold_cli/commands/run_log.py +291 -0
  20. manifold_cli/commands/runner.py +852 -0
  21. manifold_cli/commands/table.py +101 -0
  22. manifold_cli/commands/token.py +144 -0
  23. manifold_cli/dependencies.py +116 -0
  24. manifold_cli/models/__init__.py +0 -0
  25. manifold_cli/models/auth.py +49 -0
  26. manifold_cli/models/benchmark.py +87 -0
  27. manifold_cli/models/catalog.py +189 -0
  28. manifold_cli/models/credentials.py +19 -0
  29. manifold_cli/models/errors.py +121 -0
  30. manifold_cli/models/log.py +114 -0
  31. manifold_cli/models/observability.py +121 -0
  32. manifold_cli/models/organization.py +19 -0
  33. manifold_cli/models/pagination.py +59 -0
  34. manifold_cli/models/policy.py +49 -0
  35. manifold_cli/models/run.py +381 -0
  36. manifold_cli/models/runner.py +186 -0
  37. manifold_cli/models/runner_home.py +69 -0
  38. manifold_cli/models/task_queue.py +40 -0
  39. manifold_cli/models/token.py +39 -0
  40. manifold_cli/replay/__init__.py +68 -0
  41. manifold_cli/replay/log.py +814 -0
  42. manifold_cli/runner/__init__.py +72 -0
  43. manifold_cli/runner/api_retry.py +36 -0
  44. manifold_cli/runner/benchmark/__init__.py +800 -0
  45. manifold_cli/runner/benchmark/live_view.py +248 -0
  46. manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
  47. manifold_cli/runner/benchmark/recording.py +1208 -0
  48. manifold_cli/runner/benchmark/scheduling.py +256 -0
  49. manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
  50. manifold_cli/runner/benchmark/task_server.py +123 -0
  51. manifold_cli/runner/benchmark/video.py +124 -0
  52. manifold_cli/runner/benchmark_steps.py +178 -0
  53. manifold_cli/runner/cancellation.py +22 -0
  54. manifold_cli/runner/capture.py +282 -0
  55. manifold_cli/runner/container.py +370 -0
  56. manifold_cli/runner/current_dispatch.py +25 -0
  57. manifold_cli/runner/daemon.py +520 -0
  58. manifold_cli/runner/dispatch.py +38 -0
  59. manifold_cli/runner/gpu.py +182 -0
  60. manifold_cli/runner/heartbeat.py +202 -0
  61. manifold_cli/runner/loopback.py +102 -0
  62. manifold_cli/runner/loopback_bench.py +183 -0
  63. manifold_cli/runner/loopback_policy.py +39 -0
  64. manifold_cli/runner/output.py +218 -0
  65. manifold_cli/runner/pairing_acceptance.py +57 -0
  66. manifold_cli/runner/plan.py +369 -0
  67. manifold_cli/runner/policy.py +350 -0
  68. manifold_cli/runner/policy_steps.py +164 -0
  69. manifold_cli/runner/preparation.py +47 -0
  70. manifold_cli/runner/process.py +219 -0
  71. manifold_cli/runner/ram.py +17 -0
  72. manifold_cli/runner/resource_usage.py +265 -0
  73. manifold_cli/runner/served.py +218 -0
  74. manifold_cli/runner/service.py +120 -0
  75. manifold_cli/runner/side_failure.py +68 -0
  76. manifold_cli/runner/signature.py +98 -0
  77. manifold_cli/runner/source.py +66 -0
  78. manifold_cli/runner/stages.py +241 -0
  79. manifold_cli/runner/step_timing.py +229 -0
  80. manifold_cli/runner/worker.py +534 -0
  81. manifold_cli/runner/workload.py +401 -0
  82. manifold_cli/services/__init__.py +0 -0
  83. manifold_cli/services/auth.py +89 -0
  84. manifold_cli/services/benchmark.py +357 -0
  85. manifold_cli/services/organization.py +22 -0
  86. manifold_cli/services/policy.py +234 -0
  87. manifold_cli/services/run.py +304 -0
  88. manifold_cli/services/runner.py +207 -0
  89. manifold_cli/services/runner_daemon.py +146 -0
  90. manifold_cli/services/token.py +52 -0
  91. manifold_cli/settings.py +30 -0
  92. manifold_cli/transport/__init__.py +31 -0
  93. manifold_cli/transport/diagnostics.py +72 -0
  94. manifold_cli/transport/interfaces.py +69 -0
  95. manifold_cli/transport/iroh.py +317 -0
  96. manifold_cli/transport/registry.py +34 -0
  97. manifold_cli/transport/tcp.py +146 -0
  98. manifold_cli-0.0.39.dist-info/METADATA +406 -0
  99. manifold_cli-0.0.39.dist-info/RECORD +102 -0
  100. manifold_cli-0.0.39.dist-info/WHEEL +4 -0
  101. manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
  102. manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,1208 @@
1
+ """Turn a benchmark's replay logs into one Rerun recording per episode.
2
+
3
+ A benchmark worker writes a replay log per episode into the results directory
4
+ the runner mounts, and the runner reads it here. Nothing in this module knows
5
+ which benchmark wrote a log: the scene is whatever bodies the log carries, and
6
+ the views are derived from the channels it declares (ADR 0008).
7
+
8
+ One recording is one episode, stamped with a store id of its own (ADR 0007): the
9
+ recording id names the run as well as the episode, so two runs' episode 4 stay
10
+ two recordings when a viewer is handed both.
11
+
12
+ Every call into the Rerun SDK is made from a child process `RecordingWriter`
13
+ spawns and waits for, one per recording. Two things force that. The SDK deadlocks
14
+ where two threads build a recording at once, and a runner builds a task's
15
+ recordings on several threads at once. And a `RecordingStream` cannot be closed:
16
+ its sink thread and chunk batcher outlive the call whatever the caller does --
17
+ `__exit__`, an explicit `disconnect()` and dropping the last reference all leave
18
+ them running -- so a daemon that built one recording per episode in its own
19
+ process accumulated two threads and their buffers per episode until the box ran
20
+ out of memory. A child process ends them by exiting, and takes an episode's
21
+ decoded frames with it.
22
+ """
23
+
24
+ import json
25
+ import multiprocessing as mp
26
+ import re
27
+ import threading
28
+ from collections.abc import Iterable
29
+ from concurrent.futures import ThreadPoolExecutor
30
+ from dataclasses import dataclass, field, replace
31
+ from datetime import timedelta
32
+ from multiprocessing import connection
33
+ from pathlib import Path
34
+ from typing import Any
35
+ from uuid import UUID
36
+
37
+ import numpy as np
38
+ import rerun as rr
39
+ import rerun.blueprint as rrb
40
+ from tenacity import wait_exponential
41
+
42
+ from manifold_cli.replay import (
43
+ REPLAY_LOG_SUFFIX,
44
+ Channel,
45
+ ChannelKind,
46
+ Pose,
47
+ ReplayLog,
48
+ ReplayStep,
49
+ iter_replay_steps,
50
+ read_replay_head,
51
+ )
52
+ from manifold_cli.runner.benchmark.video import CameraVideo
53
+
54
+ REPLAY_UPLOAD_EXECUTOR = ThreadPoolExecutor(thread_name_prefix="manifold-replay-upload")
55
+
56
+ RECORDING_UPLOAD_ATTEMPTS = 6
57
+ RECORDING_UPLOAD_RETRY_WAIT = wait_exponential(multiplier=5, max=60)
58
+
59
+ # Where the entities a log becomes live. The blueprint addresses these, so the
60
+ # two have to agree and only this module writes either.
61
+ _BODIES_ROOT = "world/bodies"
62
+ _CAMERAS_ROOT = "cameras"
63
+ _LOGS_ROOT = "logs"
64
+ _MANIFOLD_EVENTS_ENTITY = "manifold_events"
65
+ _MANIFOLD_SCENE_ENTITY = "manifold_scene"
66
+ _STEP_TIMELINE = "step"
67
+
68
+ # A benchmark joins everything one step is worth into a single channel value, so a
69
+ # step that saw four things carries them as one string. Splitting on this puts each
70
+ # back on a row of its own.
71
+ #
72
+ # === TODO: Currently overfitting to RoboLab ================================
73
+ #
74
+ # This separator is a convention rather than a declaration. Every benchmark
75
+ # writing an events channel today joins with " / " - robolab, arena and libero all
76
+ # do - but nothing in the format says so, and one that joined with anything else
77
+ # silently gets a single row per step with no error to notice. Declaring it in the
78
+ # log's header beside the channel would tell the reader rather than leave it
79
+ # assuming (ADR 0004, the log describes itself) - a format change, not taken while
80
+ # the three benchmarks that write this channel all happen to agree.
81
+ _EVENT_SEPARATOR = " / "
82
+
83
+ # Events split out of one step's value all carry that step's index, and a query
84
+ # indexed by `step` returns the first row and hides the rest behind it. This second
85
+ # index is monotonic across the episode, so every event stays addressable.
86
+ _EVENT_TIMELINE = "event"
87
+
88
+ # A line may open with its own severity, which belongs in the viewer's level column
89
+ # rather than in the sentence. Only names Rerun itself recognises are read, and only
90
+ # as the whole bracketed word: a line opening `[gripper]` or `[2/3]` matches nothing
91
+ # here, so it keeps its bracket and takes the default rather than losing a word to a
92
+ # level nothing renders. A line opening with none is left as the benchmark wrote it.
93
+ #
94
+ # === TODO: Currently overfitting to RoboLab ================================
95
+ #
96
+ # The bracketed-tag convention is undeclared, so this is the reader holding a
97
+ # formatting rule on a benchmark's behalf - the thing ADR 0008 says this module does
98
+ # not do. The narrowness above makes it safe today rather than correct: a benchmark
99
+ # that legitimately opens a line with "[INFO]" has it removed from the body and has
100
+ # no way to say it meant it. Fixing both halves means the channel declaring what its
101
+ # tags mean rather than the reader knowing - which is a format change, and deliberately
102
+ # not taken: with one benchmark emitting levels the declaration would carry nothing a
103
+ # reader could not already assume. Revisit when a second one wants them.
104
+ _LEVEL_TAG = re.compile(r"^\[(INFO|WARN|TRACE)\]\s*")
105
+ _LEVELS = {
106
+ "INFO": rr.TextLogLevel.INFO,
107
+ "WARN": rr.TextLogLevel.WARN,
108
+ "TRACE": rr.TextLogLevel.TRACE,
109
+ }
110
+
111
+ # What a row that declared no severity gets. TRACE rather than nothing, for two
112
+ # reasons. Rerun resolves a component to its most recent value, so a row writing no
113
+ # level at all renders with the level of whatever event came before it - a gripper
114
+ # line inheriting the WARN of the drop above it, reporting a mishap it never
115
+ # reported. And TRACE is a level the viewer knows, where an empty one is not, so the
116
+ # mechanics a benchmark narrates can be styled and filtered away rather than merely
117
+ # being blank: "very low priority, often extremely verbose" is what they are.
118
+ #
119
+ # === TODO: Currently overfitting to RoboLab ================================
120
+ #
121
+ # TRACE reads RoboLab's particular mix, where the untagged lines are gripper
122
+ # mechanics worth filtering away and the tagged ones are its own diary. A benchmark
123
+ # whose every line is meaningful gets all of them buried at the lowest level the
124
+ # viewer has. It would travel with whatever declares the tag convention, so a channel
125
+ # could say what its untagged lines are worth - the same deferred format change the
126
+ # tag TODO above describes, and deferred for the same reason.
127
+ _DEFAULT_LEVEL = "TRACE"
128
+
129
+ # The 3D view's opening camera sits this many scene-diagonals back from the
130
+ # centre it orbits, which frames a tabletop without knowing it is one.
131
+ _EYE_DIAGONALS = 1.1
132
+
133
+ # How the Overview's height is split between the camera row, the scene and one
134
+ # text panel. The scene is what a replay is of, so it takes the most of the three.
135
+ # The last two split the width between that column and the plots beside it.
136
+ _CAMERA_SHARE = 2.5
137
+ _SCENE_SHARE = 3.0
138
+ _TEXT_SHARE = 1.2
139
+ _SCENE_COLUMN_SHARE = 1.4
140
+ _PLOT_COLUMN_SHARE = 1.0
141
+
142
+ # How many of the overview group's plots go side by side before the next row,
143
+ # where the group numbered none of its own. A vector is three components, so a
144
+ # row holds one.
145
+ _PLOTS_PER_ROW = 3
146
+
147
+ # How much taller a row of plots is per plot beyond the first. Plots on one row
148
+ # split its width, so a row of three is drawn narrow and is given the height back
149
+ # to keep a curve legible; a row of one already has the width.
150
+ _ROW_SHARE_PER_PLOT = 0.25
151
+
152
+ # Keep the flush timeout below the process deadline, so that a flush expiring
153
+ # while the parent still waits raises in `_build_recording` and reaches it as a
154
+ # message rather than as a killed child. Only a flush starting within the
155
+ # difference between the two bounds expires first: the process deadline is what
156
+ # ends a build whose later flush stalls.
157
+ RERUN_FLUSH_TIMEOUT = timedelta(minutes=4)
158
+ RECORDING_TIMEOUT = timedelta(minutes=5)
159
+ RECORDING_KILL_TIMEOUT = timedelta(seconds=10)
160
+
161
+ # How many recordings one runner process builds at once, so two runners on one
162
+ # box build eight. A build takes one CPU core for x264 or one NVENC session per
163
+ # camera, shared with the benchmark, and a GeForce card opens eight NVENC sessions.
164
+ MAX_CONCURRENT_RECORDINGS = 4
165
+ _RECORDING_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RECORDINGS)
166
+
167
+ # `Process.start` reaps every finished `multiprocessing` child, then stores its
168
+ # exit code. Starting a child, and joining it to read its exit code, both hold
169
+ # this lock, so a read never runs between the reap and the store.
170
+ _REAP_LOCK = threading.Lock()
171
+
172
+ # Maximum pose difference for a body to count as still.
173
+ _STILL_METRES = 1e-6
174
+
175
+ # Minimum body-to-moving-scene diagonal ratio for scenery.
176
+ _SCENERY_DIAGONALS = 3.0
177
+
178
+
179
+ def replay_logs_in(results_dir: Path) -> list[Path]:
180
+ """Every replay log a benchmark worker left in `results_dir`.
181
+
182
+ Empty is the ordinary answer: a benchmark that writes no logs still runs, and
183
+ its episodes get camera and plot views from nothing at all (ADR 0003).
184
+ """
185
+ return sorted(results_dir.glob(f"*{REPLAY_LOG_SUFFIX}"))
186
+
187
+
188
+ def _event_rows(line: str) -> list[tuple[str, str]]:
189
+ """One channel value as the rows it is worth: each body, and the level it declared.
190
+
191
+ A tag that is read is REMOVED from the body, because a level rendered in its own
192
+ column and repeated at the head of the sentence is the same fact written twice.
193
+
194
+ A row that declared no level gets `_DEFAULT_LEVEL` rather than nothing, so it is
195
+ written rather than left to inherit.
196
+ """
197
+ rows: list[tuple[str, str]] = []
198
+ for piece in (p.strip() for p in line.split(_EVENT_SEPARATOR)):
199
+ if not piece:
200
+ continue
201
+ tag = _LEVEL_TAG.match(piece)
202
+ if tag is None:
203
+ rows.append((piece, _DEFAULT_LEVEL))
204
+ else:
205
+ rows.append((piece[tag.end() :], tag.group(1)))
206
+ return rows
207
+
208
+
209
+ class RecordingError(Exception):
210
+ """A recording could not be built.
211
+
212
+ Carries what the child process reported, or how it died where it reported
213
+ nothing -- a segfault or an OOM kill leaves an exit code and no message.
214
+ """
215
+
216
+
217
+ class RecordingTimeoutError(RecordingError):
218
+ """A recording build did not finish within `RECORDING_TIMEOUT`.
219
+
220
+ Separate from its base so that a caller can stop rebuilding the log: a build
221
+ that expired takes the whole deadline again on the next attempt, where a
222
+ build that failed may succeed.
223
+ """
224
+
225
+
226
+ class RecordingWriter:
227
+ """Builds an episode's recording in a child process.
228
+
229
+ The work happens in `_build_recording` in a spawned child rather than here:
230
+ see this module's docstring for why a recording stream cannot be built in a
231
+ long-lived process. The parent reads none of the log, so an episode's frames
232
+ never enter the daemon's own memory.
233
+ """
234
+
235
+ def write_recording(
236
+ self, log_path: Path, output_path: Path, *, run_id: UUID
237
+ ) -> None:
238
+ """Build the recording for the log at `log_path`, at `output_path`.
239
+
240
+ Takes the log's path rather than a `ReplayLog`: the child reads it, so
241
+ the frames are decoded once, in the process that is about to exit. Handing
242
+ it a parsed log would mean decoding an episode here and pickling it across
243
+ a pipe, which is the cost this exists to avoid.
244
+
245
+ Waits while `MAX_CONCURRENT_RECORDINGS` builds are running. The deadline
246
+ starts when the child does.
247
+
248
+ Raises:
249
+ RecordingTimeoutError: The child did not finish within
250
+ `RECORDING_TIMEOUT`, or stayed alive after being killed.
251
+ RecordingError: The child failed, or died without reporting why.
252
+ """
253
+ with _RECORDING_SLOTS:
254
+ self._write_in_child(log_path, output_path, run_id=run_id)
255
+
256
+ def _write_in_child(
257
+ self, log_path: Path, output_path: Path, *, run_id: UUID
258
+ ) -> None:
259
+ context = mp.get_context("spawn")
260
+ # spawn, not fork: the daemon holds an event loop, worker threads and
261
+ # sockets that a forked child would inherit half-owned.
262
+ reported = context.SimpleQueue()
263
+ child = context.Process(
264
+ target=_build_recording,
265
+ args=(str(log_path), str(output_path), run_id.hex, reported),
266
+ )
267
+ with _REAP_LOCK:
268
+ child.start()
269
+ timed_out = not connection.wait(
270
+ [child.sentinel], RECORDING_TIMEOUT.total_seconds()
271
+ )
272
+ if timed_out:
273
+ child.kill()
274
+ # Raise before reading `reported`: a child killed part of the way
275
+ # through `put` leaves the first bytes of a message in the pipe, and
276
+ # `get` waits for the rest of it without a deadline.
277
+ if not connection.wait(
278
+ [child.sentinel], RECORDING_KILL_TIMEOUT.total_seconds()
279
+ ):
280
+ raise RecordingTimeoutError(
281
+ f"the recording process for {log_path.name} did not stop "
282
+ f"within {RECORDING_KILL_TIMEOUT.total_seconds():g}s after "
283
+ "being killed"
284
+ )
285
+ # Read on the deadline path too: a child that reported its failure and
286
+ # then stalled on the way out is what the deadline catches, and the
287
+ # message is the only thing separating that from a build that was slow.
288
+ failure = None if reported.empty() else reported.get()
289
+ if timed_out:
290
+ said = "" if failure is None else f", after reporting {failure}"
291
+ raise RecordingTimeoutError(
292
+ f"the recording process for {log_path.name} exceeded "
293
+ f"{RECORDING_TIMEOUT.total_seconds():g}s{said}"
294
+ )
295
+ if failure is not None:
296
+ raise RecordingError(str(failure))
297
+ with _REAP_LOCK:
298
+ child.join()
299
+ exitcode = child.exitcode
300
+ if exitcode != 0:
301
+ raise RecordingError(
302
+ f"the recording process for {log_path.name} exited {exitcode}"
303
+ )
304
+
305
+ def render(self, log: ReplayLog, output_path: Path, *, run_id: UUID) -> None:
306
+ """Write `log` as the episode's recording at `output_path`, in this process.
307
+
308
+ `run_id` is in the store id rather than only the filename. Rerun merges two
309
+ recordings sharing one, and an episode index alone is shared by every run, so
310
+ a viewer handed episode 4 of two runs would interleave them onto one timeline.
311
+
312
+ The blueprint is flushed before the episode is logged, so it is written
313
+ ahead of it. A viewer applies a blueprint where it meets it in the stream
314
+ and draws its own default layout until then, so a layout that lands after
315
+ the frames is a layout the viewer never applies to them. Logging it first
316
+ is not enough on its own: the chunk batcher decides what reaches the sink
317
+ and when, and under load it has written the episode first.
318
+
319
+ NOTE: this is where the leak in this module's docstring lives. The stream
320
+ it opens cannot be closed, so every call costs the calling process two
321
+ threads and their buffers, permanently. Only a process about to exit may
322
+ call it, which here is a test. Everything else wants `write_recording`.
323
+
324
+ Raises:
325
+ RuntimeError: A flush did not finish within `RERUN_FLUSH_TIMEOUT`.
326
+ """
327
+ self._render(
328
+ log,
329
+ replace(_survey(log.steps), depth_ranges=depth_ranges(log)),
330
+ log.steps,
331
+ output_path,
332
+ run_id=run_id,
333
+ )
334
+
335
+ def render_file(self, log_path: Path, output_path: Path, *, run_id: UUID) -> None:
336
+ """Write the log at `log_path` as its recording, reading one step at a time.
337
+
338
+ Two cheap passes come first: one reads the poses alone, without decoding a
339
+ frame, for where the 3D view's camera starts, and one decodes the few steps
340
+ the depth colour range samples. The third pass encodes and logs each step
341
+ and drops it before reading the next, so the recording file is the only
342
+ thing that grows with the episode.
343
+
344
+ The same NOTE as `render` applies: only a process about to exit may call it,
345
+ which is `_build_recording`.
346
+ """
347
+ head = read_replay_head(log_path)
348
+ survey = _survey(iter_replay_steps(log_path, decode=lambda _: False))
349
+ stride = _range_stride(survey.steps)
350
+ sampled = iter_replay_steps(log_path, decode=lambda index: index % stride == 0)
351
+ survey = replace(survey, depth_ranges=_depth_ranges(sampled))
352
+ self._render(
353
+ head,
354
+ survey,
355
+ iter_replay_steps(log_path),
356
+ output_path,
357
+ run_id=run_id,
358
+ )
359
+
360
+ def _render(
361
+ self,
362
+ head: ReplayLog,
363
+ survey: "_Survey",
364
+ steps: Iterable[ReplayStep],
365
+ output_path: Path,
366
+ *,
367
+ run_id: UUID,
368
+ ) -> None:
369
+ stream = rr.RecordingStream(
370
+ "manifold",
371
+ recording_id=f"{run_id.hex}-episode-{head.episode_idx}",
372
+ send_properties=False,
373
+ )
374
+ stream.save(str(output_path))
375
+ stream.send_blueprint(blueprint_for(head, survey))
376
+ # Ahead of the episode rather than merely before it in call order:
377
+ # see `render` on why the batcher needs telling.
378
+ stream.flush(timeout_sec=RERUN_FLUSH_TIMEOUT.total_seconds())
379
+ self._log_cameras(stream, head)
380
+ self._log_scene(stream, head)
381
+ self._log_steps(stream, head, survey, steps)
382
+ # Blocking, and before the caller moves on: the sink thread writes the
383
+ # file, and a process that exited without waiting for it would leave
384
+ # the recording truncated.
385
+ stream.flush(timeout_sec=RERUN_FLUSH_TIMEOUT.total_seconds())
386
+
387
+ def _log_cameras(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
388
+ """Log each camera's pinhole once, so its depth frame can be projected.
389
+
390
+ Static, because intrinsics are: the header declares them for the run and the
391
+ per-step half of a calibration is the pose (ADR 0008). Without this a depth
392
+ frame is a 2D image the viewer cannot place in the scene; with it, and with
393
+ the per-step `Transform3D` below, Rerun back-projects it against the bodies.
394
+
395
+ `image_from_camera` is the 3x3 the intrinsics imply. Rerun reads a pinhole in
396
+ the same axes the log fixes for the pose — OpenCV, x right, y down, z forward
397
+ — which is `RDF`, so it is stated rather than left to the default.
398
+ """
399
+ for camera in log.cameras:
400
+ stream.log(
401
+ f"/{_CAMERAS_ROOT}/{camera.name}",
402
+ rr.Pinhole(
403
+ image_from_camera=[
404
+ [camera.fx, 0.0, camera.cx],
405
+ [0.0, camera.fy, camera.cy],
406
+ [0.0, 0.0, 1.0],
407
+ ],
408
+ width=camera.width,
409
+ height=camera.height,
410
+ camera_xyz=rr.ViewCoordinates.RDF,
411
+ ),
412
+ static=True,
413
+ )
414
+
415
+ def _log_samples(
416
+ self,
417
+ stream: rr.RecordingStream,
418
+ path: str,
419
+ samples: list[tuple[int, bytes]],
420
+ events: int,
421
+ ) -> None:
422
+ """Log each sample at the step of the frame it holds, on `step` alone.
423
+
424
+ An encoder can hand a sample back a few frames after its own, so each is
425
+ stamped with its frame's step, and the step being logged is set again by
426
+ the caller's next step. The `event` timeline is left off the samples and
427
+ restored after them, so the rows that follow keep the event they had.
428
+ """
429
+ if not samples:
430
+ return
431
+ stream.disable_timeline(_EVENT_TIMELINE)
432
+ for step, sample in samples:
433
+ stream.set_time(_STEP_TIMELINE, sequence=step)
434
+ stream.log(path, rr.VideoStream.from_fields(sample=sample))
435
+ if events > 0:
436
+ stream.set_time(_EVENT_TIMELINE, sequence=events - 1)
437
+
438
+ def _log_scene(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
439
+ """Log every body's geometry once, as static data under its own entity.
440
+
441
+ A part's pose is static and its body's is per step, and Rerun composes a
442
+ transform down the entity path, so the parts follow the body without being
443
+ written again.
444
+ """
445
+ for body in log.bodies:
446
+ for index, part in enumerate(body.parts):
447
+ path = f"{_body_path(body.name)}/{index}"
448
+ stream.log(
449
+ path,
450
+ rr.Transform3D(
451
+ translation=part.position.tolist(),
452
+ quaternion=rr.Quaternion(xyzw=part.orientation),
453
+ ),
454
+ static=True,
455
+ )
456
+ vertices = part.mesh.vertices.astype(np.float32)
457
+ faces = part.mesh.faces.astype(np.uint32)
458
+ stream.log(
459
+ path,
460
+ rr.Mesh3D(
461
+ vertex_positions=vertices,
462
+ triangle_indices=faces,
463
+ vertex_normals=_vertex_normals(vertices, faces),
464
+ albedo_factor=[round(channel * 255) for channel in part.color],
465
+ ),
466
+ static=True,
467
+ )
468
+ self._log_scene_labels(stream, log)
469
+
470
+ def _log_scene_labels(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
471
+ """Log the body labels, the fingertip pads and the step length as static JSON.
472
+
473
+ Each body and pad is keyed by the entity path of its pose, so a reader matches
474
+ it to the poses without knowing how a body name becomes a path. A log that
475
+ labels nothing logs nothing.
476
+ """
477
+ labels = {
478
+ _body_path(body.name): {"role": body.role, "object": body.object}
479
+ for body in log.bodies
480
+ if body.role is not None
481
+ }
482
+ if not labels and not log.finger_pads:
483
+ return
484
+ scene = {
485
+ "bodies": labels,
486
+ "finger_pads": [
487
+ {"body": _body_path(pad.body), "offset": pad.offset.tolist()}
488
+ for pad in log.finger_pads
489
+ ],
490
+ "step_seconds": log.step_seconds,
491
+ }
492
+ stream.log(
493
+ f"/{_MANIFOLD_SCENE_ENTITY}",
494
+ rr.AnyValues(scene=json.dumps(scene)),
495
+ static=True,
496
+ )
497
+
498
+ def _log_steps(
499
+ self,
500
+ stream: rr.RecordingStream,
501
+ log: ReplayLog,
502
+ survey: "_Survey",
503
+ steps: Iterable[ReplayStep],
504
+ ) -> None:
505
+ """Log each step's body poses and channel values on the `step` timeline.
506
+
507
+ A body that never moves is logged once, as a static pose, rather than on every
508
+ step.
509
+
510
+ A depth frame is logged as the array the reader decoded. A colour frame is
511
+ sent to its camera's H.264 encoder, one per camera, opened at the camera's
512
+ first frame, and the samples it finishes are logged at once. The codec is
513
+ static on the camera's entity.
514
+
515
+ A text channel is logged only where its line changed. Rerun renders text as a
516
+ log, one row per entry, so a channel carrying the same line every step - a task
517
+ instruction, say - gives a panel of one sentence repeated as many times as the
518
+ episode is long, and anything else the benchmark said is lost among the copies.
519
+
520
+ A changed line is then split into the rows it is worth, each stamped on the
521
+ `event` timeline as well as `step` and carrying whatever level it declared.
522
+ The dedup above still compares the JOINED value, because what it exists to
523
+ suppress is a channel repeating itself, and splitting must not change which
524
+ steps it treats as having spoken.
525
+ """
526
+ # Each plot is one entity, and each channel one series of it.
527
+ members: dict[str, list[str]] = {}
528
+ slots: dict[str, tuple[str, int]] = {}
529
+ for channel in log.channels:
530
+ if channel.kind is not ChannelKind.SCALAR:
531
+ continue
532
+ origin = _scalar_view(channel)
533
+ names = members.setdefault(origin, [])
534
+ slots[channel.name] = (origin, len(names))
535
+ leaf = channel.name.rsplit("/", 1)[-1]
536
+ names.append(f"{origin}/{leaf}" if "/" in channel.name else origin)
537
+ series = {
538
+ origin: [_series_name(path, members) for path in paths]
539
+ for origin, paths in members.items()
540
+ }
541
+ said: dict[str, str] = {}
542
+ # Monotonic across the episode, so no event hides behind another.
543
+ events = 0
544
+ # The last camera pose written, so an unmoved camera is not written again.
545
+ placed: dict[str, tuple[bytes, bytes]] = {}
546
+ # Once for the episode, ahead of the frames it applies to.
547
+ ranges = survey.depth_ranges
548
+ videos: dict[str, CameraVideo] = {}
549
+ poses: dict[str, _Column] = {}
550
+ scalars: dict[str, _Column] = {}
551
+ still = set(survey.first_poses) - survey.moving
552
+ for name in still:
553
+ pose = survey.first_poses[name]
554
+ stream.log(
555
+ _body_path(name),
556
+ rr.Transform3D(
557
+ translation=pose.position.tolist(),
558
+ quaternion=rr.Quaternion(xyzw=pose.orientation),
559
+ ),
560
+ static=True,
561
+ )
562
+ for step in steps:
563
+ stream.set_time(_STEP_TIMELINE, sequence=step.index)
564
+ event = events - 1 if events > 0 else None
565
+ for name, pose in step.poses.items():
566
+ if name in still:
567
+ continue
568
+ column = poses.setdefault(_body_path(name), _Column())
569
+ column.add(step.index, event, (pose.position, pose.orientation))
570
+ # A camera's pose belongs on the same entity as its frames, so the
571
+ # transform composes down to the image the way a body's composes down to
572
+ # its parts.
573
+ #
574
+ # Logged only where it changed, like a text channel below: the reader
575
+ # hands us every camera on every step (it fills a log's sparse
576
+ # extrinsics forward), and writing all of them would put the rows back
577
+ # that the log went to the trouble of not storing. Rerun holds a
578
+ # transform until the next one, so a bolted-down camera keeps its pose
579
+ # for the whole episode off a single row.
580
+ for name, pose in step.extrinsics.items():
581
+ seen = placed.get(name)
582
+ current = (pose.position.tobytes(), pose.orientation.tobytes())
583
+ if seen == current:
584
+ continue
585
+ placed[name] = current
586
+ stream.log(
587
+ f"/{_CAMERAS_ROOT}/{name}",
588
+ rr.Transform3D(
589
+ translation=pose.position.tolist(),
590
+ quaternion=rr.Quaternion(xyzw=pose.orientation),
591
+ ),
592
+ )
593
+ for name, image in step.images.items():
594
+ if image.dtype != np.uint8:
595
+ stream.log(
596
+ f"/{_CAMERAS_ROOT}/{name}",
597
+ self._as_depth_image(image, ranges.get(name)),
598
+ )
599
+ rows: dict[str, np.ndarray] = {}
600
+ for name, value in step.scalars.items():
601
+ if name not in slots:
602
+ raise ValueError(f"scalar channel {name!r} is not in the header")
603
+ origin, slot = slots[name]
604
+ row = rows.get(origin)
605
+ if row is None:
606
+ row = rows[origin] = np.full(len(series[origin]), np.nan)
607
+ row[slot] = value
608
+ for origin, row in rows.items():
609
+ scalars.setdefault(origin, _Column()).add(step.index, event, row)
610
+ # === TODO: Currently overfitting to RoboLab ====================
611
+ #
612
+ # Every TEXT channel goes through the event splitter, including one
613
+ # carrying a single sentence rather than a list of events. An
614
+ # `instruction` that contains " / " is split into rows that are not
615
+ # events, and one opening with a bracketed level loses it from the body.
616
+ # Latent rather than live: no `instruction` in any recorded run does
617
+ # either. Only a channel that declares itself a stream of events should be
618
+ # split; the rest should be logged whole, which is what they were before.
619
+ for name, line in step.text.items():
620
+ if said.get(name) == line:
621
+ continue
622
+ said[name] = line
623
+ for body, level in _event_rows(line):
624
+ stream.set_time(_EVENT_TIMELINE, sequence=events)
625
+ events += 1
626
+ stream.log(
627
+ f"/{_LOGS_ROOT}/{name}",
628
+ rr.TextLog(body, level=_LEVELS[level]),
629
+ )
630
+ # The Manifold events, as JSON on their own entity: grading reads them,
631
+ # and keeping them out of the logs keeps them out of the text panel.
632
+ if step.events:
633
+ stream.log(
634
+ f"/{_MANIFOLD_EVENTS_ENTITY}",
635
+ rr.AnyValues(events=json.dumps(list(step.events))),
636
+ )
637
+ # Last in the step: a sample moves the `step` timeline to its own frame.
638
+ for name, image in step.images.items():
639
+ if image.dtype != np.uint8:
640
+ continue
641
+ path = f"/{_CAMERAS_ROOT}/{name}"
642
+ video = videos.get(name)
643
+ if video is None:
644
+ stream.log(
645
+ path, rr.VideoStream(codec=rr.VideoCodec.H264), static=True
646
+ )
647
+ video = videos[name] = CameraVideo(name)
648
+ self._log_samples(stream, path, video.encode(step.index, image), events)
649
+ for name, video in videos.items():
650
+ self._log_samples(
651
+ stream, f"/{_CAMERAS_ROOT}/{name}", video.finish(), events
652
+ )
653
+ for path, column in poses.items():
654
+ for indexes, values in column.parts():
655
+ stream.send_columns(
656
+ path,
657
+ indexes=indexes,
658
+ columns=rr.Transform3D.columns(
659
+ translation=np.stack([position for position, _ in values]),
660
+ quaternion=np.stack([orientation for _, orientation in values]),
661
+ ),
662
+ )
663
+ for path, column in scalars.items():
664
+ stream.log(path, rr.SeriesLines(names=series[path]), static=True)
665
+ for indexes, values in column.parts():
666
+ stacked = np.stack(values).astype(np.float64)
667
+ stream.send_columns(
668
+ path,
669
+ indexes=indexes,
670
+ columns=rr.Scalars.columns(scalars=stacked.reshape(-1)).partition(
671
+ [stacked.shape[1]] * stacked.shape[0]
672
+ ),
673
+ )
674
+
675
+ def _as_depth_image(
676
+ self, array: np.ndarray, depth_range: tuple[float, float] | None = None
677
+ ) -> rr.DepthImage:
678
+ """A depth frame, in floating-point metres as the log carries it.
679
+
680
+ `rr.DepthImage` stores the array's dtype as its format.
681
+
682
+ `depth_range` fixes the colour map for the whole episode. Without it the
683
+ viewer ranges each frame on its own min and max, and a wrist camera's far
684
+ bound moves as the gripper does — measured at 3.2x between the widest and
685
+ narrowest frame of one episode, and 163% in a single step. The effect is a
686
+ depth view whose colours change fast while the geometry barely moves, which
687
+ reads as the depth itself being unstable.
688
+ """
689
+ return rr.DepthImage(array, meter=1.0, depth_range=depth_range)
690
+
691
+
692
+ # Pixels sampled per depth frame, and the most frames sampled, when fixing a colour
693
+ # range. A percentile does not need every pixel of every step: half a percent of a few
694
+ # hundred thousand samples is already stable well inside a millimetre, and a 520-step
695
+ # episode's depth is tens of millions of values.
696
+ _RANGE_SAMPLE_STRIDE = 7
697
+ _RANGE_MAX_FRAMES = 64
698
+
699
+ # The percentiles the fixed colour range spans, chosen by measuring the alternatives
700
+ # rather than by argument. See `depth_ranges`.
701
+ _RANGE_PERCENTILES = (0.5, 99.5)
702
+
703
+
704
+ def _build_recording(
705
+ log_path: str, output_path: str, run_id_hex: str, reported: Any
706
+ ) -> None:
707
+ """Read one log and write its recording. Runs in a child process.
708
+
709
+ Module level and taking only strings, because `spawn` pickles the target and
710
+ its arguments to reach a fresh interpreter.
711
+
712
+ A failure is put on `reported` rather than raised: a traceback in a child
713
+ reaches the parent's journal as a bare exit code otherwise, and the caller
714
+ keeps a log eligible on the strength of the message.
715
+ """
716
+ try:
717
+ RecordingWriter().render_file(
718
+ Path(log_path),
719
+ Path(output_path),
720
+ run_id=UUID(run_id_hex),
721
+ )
722
+ # Broad: this is the child's one channel to the parent, and an exception it
723
+ # let escape would reach the journal as a bare exit code.
724
+ except Exception as exc:
725
+ reported.put(f"{type(exc).__name__}: {exc}")
726
+
727
+
728
+ @dataclass
729
+ class _Column:
730
+ """One entity's rows, each with its step and the event in effect at that step."""
731
+
732
+ steps: list[int] = field(default_factory=list)
733
+ events: list[int] = field(default_factory=list)
734
+ values: list[Any] = field(default_factory=list)
735
+
736
+ def add(self, step: int, event: int | None, value: Any) -> None:
737
+ self.steps.append(step)
738
+ self.values.append(value)
739
+ if event is not None:
740
+ self.events.append(event)
741
+
742
+ def parts(self) -> list[tuple[list[rr.TimeColumn], list[Any]]]:
743
+ """The rows as time columns and values, split where the event time starts.
744
+
745
+ A row logged before the first event carries no event time, and a column
746
+ holds a time for every row or for none.
747
+ """
748
+ split = len(self.steps) - len(self.events)
749
+ parts = []
750
+ if split > 0:
751
+ steps = rr.TimeColumn(_STEP_TIMELINE, sequence=self.steps[:split])
752
+ parts.append(([steps], self.values[:split]))
753
+ if self.events:
754
+ steps = rr.TimeColumn(_STEP_TIMELINE, sequence=self.steps[split:])
755
+ events = rr.TimeColumn(_EVENT_TIMELINE, sequence=self.events)
756
+ parts.append(([steps, events], self.values[split:]))
757
+ return parts
758
+
759
+
760
+ @dataclass
761
+ class _Survey:
762
+ """What the layout and the depth colour map need from the whole episode.
763
+
764
+ Small whatever the episode's length: the first step's poses, the names of the
765
+ bodies that move, the step count and one range per depth channel.
766
+ """
767
+
768
+ first_poses: dict[str, Pose]
769
+ moving: set[str]
770
+ steps: int
771
+ depth_ranges: dict[str, tuple[float, float]] = field(default_factory=dict)
772
+
773
+
774
+ def _survey(steps: Iterable[ReplayStep]) -> _Survey:
775
+ """Walk the steps once for the first poses, the bodies that move and the count."""
776
+ first: dict[str, Pose] | None = None
777
+ moving: set[str] = set()
778
+ count = 0
779
+ for step in steps:
780
+ count += 1
781
+ if first is None:
782
+ first = dict(step.poses)
783
+ continue
784
+ moving.update(
785
+ name
786
+ for name, pose in step.poses.items()
787
+ if name in first
788
+ and name not in moving
789
+ and not _same_pose(pose, first[name])
790
+ )
791
+ return _Survey(first_poses=first or {}, moving=moving, steps=count)
792
+
793
+
794
+ def _range_stride(steps: int) -> int:
795
+ return max(1, -(-steps // _RANGE_MAX_FRAMES))
796
+
797
+
798
+ def depth_ranges(log: ReplayLog) -> dict[str, tuple[float, float]]:
799
+ """One fixed colour range per depth channel, spanning the 0.5-99.5 percentiles.
800
+
801
+ Fixed for the episode on purpose: a range recomputed per frame is what makes a
802
+ depth view's colours move while its geometry does not (see `_as_depth_image`).
803
+
804
+ Percentiles rather than the extremes or a multiple of sigma, because the choice
805
+ was measured. Over the first two episodes of all four suites of the 80-episode
806
+ sweep, counting how many of a 256-entry colour ramp the data actually exercises
807
+ (the entropy of the quantisation, as 2**bits):
808
+
809
+ | rule | agentview | wrist | clipped |
810
+ | --------------- | --------- | ----- | ------- |
811
+ | min/max | 112 | 103 | 0.00% |
812
+ | mean +/- 3 sigma| 112 | 160 | 0.04% |
813
+ | 1/99 | 112 | 186 | 1.87% |
814
+ | **0.5/99.5** | **113** | 182 | 0.87% |
815
+
816
+ The scene camera is insensitive: its depth spans 0.68-3.07 m with no tail, so
817
+ every rule lands within one level of every other. The WRIST camera is where the
818
+ range is won or lost. A handful of rays slip past the gripper into the far scene
819
+ and stretch its extremes to 1.01 m when the surfaces that matter sit inside
820
+ 0.36 m, so min/max spends two thirds of the ramp on a few dozen pixels and
821
+ leaves 103 levels for the hand. Trimming half a percent off each end recovers
822
+ 182.
823
+
824
+ 1/99 buys four more levels on the wrist for double the clipping and one fewer on
825
+ the scene view, which is not a trade worth making. Nothing below 2/98 is
826
+ competitive.
827
+
828
+ The limit worth knowing: this trims a tail only while the tail is THINNER than
829
+ the half percent being cut. A camera where more than 0.5% of pixels sit far out
830
+ puts the 99.5th percentile inside the tail, and the range degrades gracefully to
831
+ the raw extremes rather than misbehaving. LIBERO's wrist tail is about two rays
832
+ in a thousand, well inside that.
833
+
834
+ No clamp to the observed extremes is needed, unlike the sigma rule this replaces:
835
+ a percentile is inside the data by construction, where `mean - 3 sigma` landed
836
+ below zero on every camera measured and spent part of the ramp on distances no
837
+ pixel can reach.
838
+ """
839
+ # Subsample frames as well as pixels. A percentile over the episode does not need
840
+ # every step, and a long episode's samples would otherwise be tens of megabytes.
841
+ return _depth_ranges(log.steps[:: _range_stride(len(log.steps))])
842
+
843
+
844
+ def _depth_ranges(steps: Iterable[ReplayStep]) -> dict[str, tuple[float, float]]:
845
+ samples: dict[str, list[np.ndarray]] = {}
846
+ for step in steps:
847
+ for name, frame in step.images.items():
848
+ if frame.dtype == np.uint8:
849
+ continue
850
+ flat = np.asarray(frame, dtype=np.float32).reshape(-1)
851
+ values = flat[::_RANGE_SAMPLE_STRIDE]
852
+ finite = values[np.isfinite(values)]
853
+ if finite.size:
854
+ samples.setdefault(name, []).append(finite)
855
+ ranges: dict[str, tuple[float, float]] = {}
856
+ for name, chunks in samples.items():
857
+ low, high = (
858
+ float(v) for v in np.percentile(np.concatenate(chunks), _RANGE_PERCENTILES)
859
+ )
860
+ # A channel with no spread — a flat test frame, or a camera staring at one
861
+ # surface — collapses to a single value. Omit it rather than hand the viewer
862
+ # an empty range: there is no flicker to prevent on a frame that never
863
+ # changes, and a zero-width range is a worse instruction than none.
864
+ if high > low:
865
+ ranges[name] = (low, high)
866
+ return ranges
867
+
868
+
869
+ def blueprint_for(log: ReplayLog, survey: _Survey | None = None) -> rrb.Blueprint:
870
+ """The view layout this log's channels imply.
871
+
872
+ An Overview tab holds the cameras, the scene, the text and the plots of the
873
+ group the header named; every other scalar group becomes a tab of its own,
874
+ named by the group. The named group does not get a tab: its plots are already
875
+ on the Overview. Nothing here names a channel (ADR 0008), so a benchmark
876
+ chooses its own layout by naming channels, one of their groups and the rows
877
+ they sit on.
878
+ """
879
+ if survey is None:
880
+ survey = _survey(log.steps)
881
+ views = _scalar_views(log)
882
+ overview = (
883
+ views.pop(log.overview_group, []) if log.overview_group is not None else []
884
+ )
885
+ tabs: list[Any] = [_overview_tab(log, survey, overview)]
886
+ for group, entries in views.items():
887
+ tabs.append(_plot_grid(entries, wrap=1, name=group))
888
+ return rrb.Blueprint(
889
+ rrb.Tabs(*tabs, active_tab=0),
890
+ rrb.SelectionPanel(state="collapsed"),
891
+ rrb.BlueprintPanel(state="hidden"),
892
+ rrb.TimePanel(state="collapsed"),
893
+ )
894
+
895
+
896
+ def _overview_tab(
897
+ log: ReplayLog, survey: _Survey, overview_entries: list[tuple[str, int | None]]
898
+ ) -> Any:
899
+ """The cameras over the scene and the text, beside the overview group's plots.
900
+
901
+ The cameras come first because they are what the policy saw, and the scene
902
+ under them is the same moment from outside it.
903
+
904
+ One column when the header named no group, which is every log that says
905
+ nothing about what is worth watching while the episode plays.
906
+
907
+ No scene when no body has geometry: a 3D view of poses alone draws nothing.
908
+ """
909
+ left: list[Any] = []
910
+ shares: list[float] = []
911
+ cameras = [
912
+ rrb.Spatial2DView(origin=f"/{_CAMERAS_ROOT}/{channel.name}", name=channel.name)
913
+ for channel in log.channels
914
+ if channel.kind is ChannelKind.IMAGE
915
+ ]
916
+ if len(cameras) > 0:
917
+ left.append(rrb.Horizontal(*cameras))
918
+ shares.append(_CAMERA_SHARE)
919
+ if any(len(body.parts) > 0 for body in log.bodies):
920
+ left.append(
921
+ rrb.Spatial3DView(
922
+ origin=f"/{_BODIES_ROOT}",
923
+ name="scene",
924
+ contents=[
925
+ "+ $origin/**",
926
+ *[
927
+ f"- $origin/{_entity_name(name)}/**"
928
+ for name in _scenery(log, survey)
929
+ ],
930
+ ],
931
+ background=rrb.Background(kind=rrb.BackgroundKind.GradientBright),
932
+ line_grid=rrb.LineGrid3D(spacing=0.05),
933
+ eye_controls=_eye_for(log, survey),
934
+ )
935
+ )
936
+ shares.append(_SCENE_SHARE)
937
+ for channel in log.channels:
938
+ if channel.kind is ChannelKind.TEXT:
939
+ left.append(
940
+ rrb.TextLogView(
941
+ origin=f"/{_LOGS_ROOT}/{channel.name}", name=channel.name
942
+ )
943
+ )
944
+ shares.append(_TEXT_SHARE)
945
+ if len(overview_entries) == 0:
946
+ return rrb.Vertical(*left, row_shares=shares, name="Overview")
947
+ return rrb.Horizontal(
948
+ rrb.Vertical(*left, row_shares=shares),
949
+ _plot_grid(overview_entries, wrap=_PLOTS_PER_ROW),
950
+ column_shares=[_SCENE_COLUMN_SHARE, _PLOT_COLUMN_SHARE],
951
+ name="Overview",
952
+ )
953
+
954
+
955
+ def _plot_grid(
956
+ entries: list[tuple[str, int | None]], *, wrap: int, name: str | None = None
957
+ ) -> Any:
958
+ """One group's plots, in the rows it numbered or in rows of `wrap`.
959
+
960
+ A row of one plot is the plot itself: it has the grid's whole width already,
961
+ and a container holding it alone would only be a level to walk through.
962
+ """
963
+ rows = _plot_rows(entries, wrap=wrap)
964
+ return rrb.Vertical(
965
+ *[_plot_row(row) for row in rows],
966
+ row_shares=[1.0 + _ROW_SHARE_PER_PLOT * (len(row) - 1) for row in rows],
967
+ name=name,
968
+ )
969
+
970
+
971
+ def _plot_rows(entries: list[tuple[str, int | None]], *, wrap: int) -> list[list[str]]:
972
+ """The plots of one group, grouped into the rows they are drawn on.
973
+
974
+ A group numbering its rows is laid out as it asked, in ascending row number.
975
+ Which of its scalars belong side by side is the benchmark's to say: three
976
+ position axes are one vector and a gripper state is a series on its own, and a
977
+ channel list distinguishes neither.
978
+
979
+ A group numbering none is wrapped into rows of `wrap` instead, which is what
980
+ every group that says nothing about its layout gets. The writer refuses a
981
+ group numbering only some of its channels, so the two cases are all of them.
982
+ """
983
+ numbered = [(origin, row) for origin, row in entries if row is not None]
984
+ if len(entries) > 0 and len(numbered) == len(entries):
985
+ rows: dict[int, list[str]] = {}
986
+ for origin, row in numbered:
987
+ rows.setdefault(row, []).append(origin)
988
+ return [rows[number] for number in sorted(rows)]
989
+ origins = [origin for origin, _ in entries]
990
+ return [origins[at : at + wrap] for at in range(0, len(origins), wrap)]
991
+
992
+
993
+ def _plot_row(origins: list[str]) -> Any:
994
+ """One row of plots, side by side."""
995
+ views = [
996
+ rrb.TimeSeriesView(origin=origin, name=origin.rsplit("/", 1)[-1])
997
+ for origin in origins
998
+ ]
999
+ return views[0] if len(views) == 1 else rrb.Horizontal(*views)
1000
+
1001
+
1002
+ def _scalar_views(log: ReplayLog) -> dict[str, list[tuple[str, int | None]]]:
1003
+ """Each group's plots, as a view origin and the row it declared, in order.
1004
+
1005
+ Channels whose names differ only in the last segment share one plot, so
1006
+ `joints/0` to `joints/6` are seven series of a plot named `joints` and a
1007
+ channel named without a `/` is a plot of its own. A benchmark chooses between
1008
+ the two by naming its channels, and chooses the rows they sit on by numbering
1009
+ them, which is the whole of what ADR 0008 lets it decide.
1010
+
1011
+ A plot takes the row of the first channel that reaches it, since the channels
1012
+ sharing one are one series and cannot be drawn in two places.
1013
+ """
1014
+ views: dict[str, list[tuple[str, int | None]]] = {}
1015
+ for channel in log.channels:
1016
+ if channel.kind is not ChannelKind.SCALAR:
1017
+ continue
1018
+ entries = views.setdefault(_scalar_group(channel), [])
1019
+ origin = _scalar_view(channel)
1020
+ if origin not in [seen for seen, _ in entries]:
1021
+ entries.append((origin, channel.row))
1022
+ return views
1023
+
1024
+
1025
+ def _scalar_group(channel: Channel) -> str:
1026
+ """The tab a scalar channel is plotted in.
1027
+
1028
+ A channel that declares no group falls back to the first segment of its name,
1029
+ so a log that says nothing about layout still gets its scalars sorted.
1030
+ """
1031
+ return channel.group or channel.name.split("/")[0]
1032
+
1033
+
1034
+ def _scalar_view(channel: Channel) -> str:
1035
+ """The origin of the plot a scalar channel's series is drawn in and logged on.
1036
+
1037
+ Under the group, except where the group came from the name's own first
1038
+ segment - prefixing that again would name the tab twice over.
1039
+ """
1040
+ if channel.group is None:
1041
+ path = f"/{channel.name}"
1042
+ else:
1043
+ path = f"/{channel.group}/{channel.name}"
1044
+ return path.rsplit("/", 1)[0] if "/" in channel.name else path
1045
+
1046
+
1047
+ def _series_name(path: str, origins: Iterable[str]) -> str:
1048
+ """A channel's series name: its path below the outermost plot that draws it.
1049
+
1050
+ A plot draws the plots nested under it as well, so `sensor/table` stays
1051
+ distinct from another sensor's `table` there.
1052
+ """
1053
+ outer = min(
1054
+ (
1055
+ origin
1056
+ for origin in origins
1057
+ if path == origin or path.startswith(f"{origin}/")
1058
+ ),
1059
+ key=len,
1060
+ )
1061
+ return path[len(outer) + 1 :] or path.rsplit("/", 1)[-1]
1062
+
1063
+
1064
+ def _eye_for(log: ReplayLog, survey: _Survey) -> rrb.EyeControls3D:
1065
+ """An opening camera derived from the extent of what moves.
1066
+
1067
+ Orbital about the centre of that box, backed off along +x by a little over its
1068
+ diagonal and lifted half as far along +z. A hand-placed position and look
1069
+ target would be exactly the per-benchmark constant a generic reader refuses.
1070
+ """
1071
+ centre, diagonal = _scene_bounds(log, survey)
1072
+ offset = max(diagonal * _EYE_DIAGONALS, 0.5)
1073
+ return rrb.EyeControls3D(
1074
+ kind=rrb.Eye3DKind.Orbital,
1075
+ position=(centre[0] + offset, centre[1], centre[2] + offset * 0.5),
1076
+ look_target=(centre[0], centre[1], centre[2]),
1077
+ eye_up=(0.0, 0.0, 1.0),
1078
+ )
1079
+
1080
+
1081
+ def _scene_bounds(log: ReplayLog, survey: _Survey) -> tuple[np.ndarray, float]:
1082
+ """The centre and diagonal of the box around the bodies that move.
1083
+
1084
+ Over the geometry, not the body origins: a body's parts sit at offsets in its
1085
+ frame and carry vertices of their own, so a robot whose links share a base
1086
+ origin has an extent its poses alone put at zero. Part rotations are ignored
1087
+ and each part's vertex box is taken axis-aligned, which can only overstate the
1088
+ extent - the eye ends up slightly further back, never inside the scene.
1089
+
1090
+ The scenery is left out. A robosuite scene carries the room it stands in as a
1091
+ body, and a floor nine metres across gives a box the tabletop occupies a
1092
+ fortieth of, so an eye framing the whole scene opens far enough away that the
1093
+ arm and the objects are a few pixels. Every body counts when none of them
1094
+ moved, which is all a single-step episode has to offer.
1095
+ """
1096
+ if len(survey.first_poses) == 0:
1097
+ return np.zeros(3), 1.0
1098
+ moving = survey.moving
1099
+ framed = moving if len(moving) > 0 else set(survey.first_poses)
1100
+ parts = {body.name: body.parts for body in log.bodies}
1101
+ corners: list[np.ndarray] = []
1102
+ for name, pose in survey.first_poses.items():
1103
+ if name not in framed:
1104
+ continue
1105
+ origin = np.asarray(pose.position, dtype=np.float64)
1106
+ corners.append(origin)
1107
+ for part in parts.get(name, ()):
1108
+ vertices = np.asarray(part.mesh.vertices, dtype=np.float64)
1109
+ if vertices.size == 0:
1110
+ continue
1111
+ offset = origin + np.asarray(part.position, dtype=np.float64)
1112
+ corners.append(offset + vertices.min(axis=0))
1113
+ corners.append(offset + vertices.max(axis=0))
1114
+ stacked = np.stack(corners)
1115
+ low, high = stacked.min(axis=0), stacked.max(axis=0)
1116
+ return (low + high) / 2.0, float(np.linalg.norm(high - low))
1117
+
1118
+
1119
+ def _scenery(log: ReplayLog, survey: _Survey) -> list[str]:
1120
+ """The bodies the 3D view leaves out: the room the scene stands in.
1121
+
1122
+ A robosuite scene carries its room as an ordinary body, and one of its parts
1123
+ is a slab nine metres across at about the height of a raised arm. It is a
1124
+ wall to look through as well as a box to be dwarfed by, so an eye placed above
1125
+ it sees nothing but it, whatever the eye is derived from.
1126
+
1127
+ The test is size against motion: a body that never moves and is several times
1128
+ the extent of everything that does is the room. A tabletop is about the size
1129
+ of the arm working on it and stays.
1130
+
1131
+ The geometry is only dropped from the view. It stays in the recording, so a
1132
+ viewer that wants the room can put it back.
1133
+ """
1134
+ moving = survey.moving
1135
+ if len(moving) == 0:
1136
+ return []
1137
+ _, span = _scene_bounds(log, survey)
1138
+ extents = _body_extents(log)
1139
+ return [
1140
+ name
1141
+ for name in survey.first_poses
1142
+ if name not in moving and extents.get(name, 0.0) > span * _SCENERY_DIAGONALS
1143
+ ]
1144
+
1145
+
1146
+ def _body_extents(log: ReplayLog) -> dict[str, float]:
1147
+ """Each body's own diagonal, over the geometry its parts carry."""
1148
+ extents: dict[str, float] = {}
1149
+ for body in log.bodies:
1150
+ corners: list[np.ndarray] = []
1151
+ for part in body.parts:
1152
+ vertices = np.asarray(part.mesh.vertices, dtype=np.float64)
1153
+ if vertices.size == 0:
1154
+ continue
1155
+ offset = np.asarray(part.position, dtype=np.float64)
1156
+ corners.append(offset + vertices.min(axis=0))
1157
+ corners.append(offset + vertices.max(axis=0))
1158
+ if len(corners) == 0:
1159
+ continue
1160
+ stacked = np.stack(corners)
1161
+ extents[body.name] = float(
1162
+ np.linalg.norm(stacked.max(axis=0) - stacked.min(axis=0))
1163
+ )
1164
+ return extents
1165
+
1166
+
1167
+ def _same_pose(pose: Pose, other: Pose) -> bool:
1168
+ """Whether two poses are the same to within what counts as standing still."""
1169
+ return bool(
1170
+ np.allclose(pose.position, other.position, atol=_STILL_METRES)
1171
+ and np.allclose(pose.orientation, other.orientation, atol=_STILL_METRES)
1172
+ )
1173
+
1174
+
1175
+ def _vertex_normals(vertices: np.ndarray, faces: np.ndarray) -> np.ndarray:
1176
+ """A normal per vertex, averaged over the triangles meeting at it.
1177
+
1178
+ A mesh logged without them is drawn unlit, which on a scene of untextured
1179
+ parts is one flat silhouette per body and no shape at all. The format carries
1180
+ no normals (manifold-sdk ADR 0004) because they are derivable, and this is
1181
+ where they are derived.
1182
+
1183
+ Each triangle's normal is its edge cross product, whose length is twice the
1184
+ triangle's area, so accumulating them unnormalised weights every vertex by the
1185
+ area around it. A degenerate triangle contributes a zero vector and a vertex
1186
+ with nothing but those keeps the up it starts as.
1187
+ """
1188
+ normals = np.zeros_like(vertices, dtype=np.float32)
1189
+ if len(faces) == 0:
1190
+ return normals
1191
+ corners = vertices[faces.astype(np.int64)]
1192
+ facing = np.cross(
1193
+ corners[:, 1] - corners[:, 0], corners[:, 2] - corners[:, 0]
1194
+ ).astype(np.float32)
1195
+ for corner in range(3):
1196
+ np.add.at(normals, faces[:, corner].astype(np.int64), facing)
1197
+ lengths = np.linalg.norm(normals, axis=1, keepdims=True)
1198
+ return np.divide(normals, lengths, out=normals, where=lengths > 0)
1199
+
1200
+
1201
+ def _body_path(name: str) -> str:
1202
+ """The entity path of a scene body's pose."""
1203
+ return f"/{_BODIES_ROOT}/{_entity_name(name)}"
1204
+
1205
+
1206
+ def _entity_name(name: str) -> str:
1207
+ """A body name as one entity-path segment."""
1208
+ return name.replace("/", "_").replace(":", "_")