manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,1208 @@
|
|
|
1
|
+
"""Turn a benchmark's replay logs into one Rerun recording per episode.
|
|
2
|
+
|
|
3
|
+
A benchmark worker writes a replay log per episode into the results directory
|
|
4
|
+
the runner mounts, and the runner reads it here. Nothing in this module knows
|
|
5
|
+
which benchmark wrote a log: the scene is whatever bodies the log carries, and
|
|
6
|
+
the views are derived from the channels it declares (ADR 0008).
|
|
7
|
+
|
|
8
|
+
One recording is one episode, stamped with a store id of its own (ADR 0007): the
|
|
9
|
+
recording id names the run as well as the episode, so two runs' episode 4 stay
|
|
10
|
+
two recordings when a viewer is handed both.
|
|
11
|
+
|
|
12
|
+
Every call into the Rerun SDK is made from a child process `RecordingWriter`
|
|
13
|
+
spawns and waits for, one per recording. Two things force that. The SDK deadlocks
|
|
14
|
+
where two threads build a recording at once, and a runner builds a task's
|
|
15
|
+
recordings on several threads at once. And a `RecordingStream` cannot be closed:
|
|
16
|
+
its sink thread and chunk batcher outlive the call whatever the caller does --
|
|
17
|
+
`__exit__`, an explicit `disconnect()` and dropping the last reference all leave
|
|
18
|
+
them running -- so a daemon that built one recording per episode in its own
|
|
19
|
+
process accumulated two threads and their buffers per episode until the box ran
|
|
20
|
+
out of memory. A child process ends them by exiting, and takes an episode's
|
|
21
|
+
decoded frames with it.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import multiprocessing as mp
|
|
26
|
+
import re
|
|
27
|
+
import threading
|
|
28
|
+
from collections.abc import Iterable
|
|
29
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
30
|
+
from dataclasses import dataclass, field, replace
|
|
31
|
+
from datetime import timedelta
|
|
32
|
+
from multiprocessing import connection
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any
|
|
35
|
+
from uuid import UUID
|
|
36
|
+
|
|
37
|
+
import numpy as np
|
|
38
|
+
import rerun as rr
|
|
39
|
+
import rerun.blueprint as rrb
|
|
40
|
+
from tenacity import wait_exponential
|
|
41
|
+
|
|
42
|
+
from manifold_cli.replay import (
|
|
43
|
+
REPLAY_LOG_SUFFIX,
|
|
44
|
+
Channel,
|
|
45
|
+
ChannelKind,
|
|
46
|
+
Pose,
|
|
47
|
+
ReplayLog,
|
|
48
|
+
ReplayStep,
|
|
49
|
+
iter_replay_steps,
|
|
50
|
+
read_replay_head,
|
|
51
|
+
)
|
|
52
|
+
from manifold_cli.runner.benchmark.video import CameraVideo
|
|
53
|
+
|
|
54
|
+
REPLAY_UPLOAD_EXECUTOR = ThreadPoolExecutor(thread_name_prefix="manifold-replay-upload")
|
|
55
|
+
|
|
56
|
+
RECORDING_UPLOAD_ATTEMPTS = 6
|
|
57
|
+
RECORDING_UPLOAD_RETRY_WAIT = wait_exponential(multiplier=5, max=60)
|
|
58
|
+
|
|
59
|
+
# Where the entities a log becomes live. The blueprint addresses these, so the
|
|
60
|
+
# two have to agree and only this module writes either.
|
|
61
|
+
_BODIES_ROOT = "world/bodies"
|
|
62
|
+
_CAMERAS_ROOT = "cameras"
|
|
63
|
+
_LOGS_ROOT = "logs"
|
|
64
|
+
_MANIFOLD_EVENTS_ENTITY = "manifold_events"
|
|
65
|
+
_MANIFOLD_SCENE_ENTITY = "manifold_scene"
|
|
66
|
+
_STEP_TIMELINE = "step"
|
|
67
|
+
|
|
68
|
+
# A benchmark joins everything one step is worth into a single channel value, so a
|
|
69
|
+
# step that saw four things carries them as one string. Splitting on this puts each
|
|
70
|
+
# back on a row of its own.
|
|
71
|
+
#
|
|
72
|
+
# === TODO: Currently overfitting to RoboLab ================================
|
|
73
|
+
#
|
|
74
|
+
# This separator is a convention rather than a declaration. Every benchmark
|
|
75
|
+
# writing an events channel today joins with " / " - robolab, arena and libero all
|
|
76
|
+
# do - but nothing in the format says so, and one that joined with anything else
|
|
77
|
+
# silently gets a single row per step with no error to notice. Declaring it in the
|
|
78
|
+
# log's header beside the channel would tell the reader rather than leave it
|
|
79
|
+
# assuming (ADR 0004, the log describes itself) - a format change, not taken while
|
|
80
|
+
# the three benchmarks that write this channel all happen to agree.
|
|
81
|
+
_EVENT_SEPARATOR = " / "
|
|
82
|
+
|
|
83
|
+
# Events split out of one step's value all carry that step's index, and a query
|
|
84
|
+
# indexed by `step` returns the first row and hides the rest behind it. This second
|
|
85
|
+
# index is monotonic across the episode, so every event stays addressable.
|
|
86
|
+
_EVENT_TIMELINE = "event"
|
|
87
|
+
|
|
88
|
+
# A line may open with its own severity, which belongs in the viewer's level column
|
|
89
|
+
# rather than in the sentence. Only names Rerun itself recognises are read, and only
|
|
90
|
+
# as the whole bracketed word: a line opening `[gripper]` or `[2/3]` matches nothing
|
|
91
|
+
# here, so it keeps its bracket and takes the default rather than losing a word to a
|
|
92
|
+
# level nothing renders. A line opening with none is left as the benchmark wrote it.
|
|
93
|
+
#
|
|
94
|
+
# === TODO: Currently overfitting to RoboLab ================================
|
|
95
|
+
#
|
|
96
|
+
# The bracketed-tag convention is undeclared, so this is the reader holding a
|
|
97
|
+
# formatting rule on a benchmark's behalf - the thing ADR 0008 says this module does
|
|
98
|
+
# not do. The narrowness above makes it safe today rather than correct: a benchmark
|
|
99
|
+
# that legitimately opens a line with "[INFO]" has it removed from the body and has
|
|
100
|
+
# no way to say it meant it. Fixing both halves means the channel declaring what its
|
|
101
|
+
# tags mean rather than the reader knowing - which is a format change, and deliberately
|
|
102
|
+
# not taken: with one benchmark emitting levels the declaration would carry nothing a
|
|
103
|
+
# reader could not already assume. Revisit when a second one wants them.
|
|
104
|
+
_LEVEL_TAG = re.compile(r"^\[(INFO|WARN|TRACE)\]\s*")
|
|
105
|
+
_LEVELS = {
|
|
106
|
+
"INFO": rr.TextLogLevel.INFO,
|
|
107
|
+
"WARN": rr.TextLogLevel.WARN,
|
|
108
|
+
"TRACE": rr.TextLogLevel.TRACE,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
# What a row that declared no severity gets. TRACE rather than nothing, for two
|
|
112
|
+
# reasons. Rerun resolves a component to its most recent value, so a row writing no
|
|
113
|
+
# level at all renders with the level of whatever event came before it - a gripper
|
|
114
|
+
# line inheriting the WARN of the drop above it, reporting a mishap it never
|
|
115
|
+
# reported. And TRACE is a level the viewer knows, where an empty one is not, so the
|
|
116
|
+
# mechanics a benchmark narrates can be styled and filtered away rather than merely
|
|
117
|
+
# being blank: "very low priority, often extremely verbose" is what they are.
|
|
118
|
+
#
|
|
119
|
+
# === TODO: Currently overfitting to RoboLab ================================
|
|
120
|
+
#
|
|
121
|
+
# TRACE reads RoboLab's particular mix, where the untagged lines are gripper
|
|
122
|
+
# mechanics worth filtering away and the tagged ones are its own diary. A benchmark
|
|
123
|
+
# whose every line is meaningful gets all of them buried at the lowest level the
|
|
124
|
+
# viewer has. It would travel with whatever declares the tag convention, so a channel
|
|
125
|
+
# could say what its untagged lines are worth - the same deferred format change the
|
|
126
|
+
# tag TODO above describes, and deferred for the same reason.
|
|
127
|
+
_DEFAULT_LEVEL = "TRACE"
|
|
128
|
+
|
|
129
|
+
# The 3D view's opening camera sits this many scene-diagonals back from the
|
|
130
|
+
# centre it orbits, which frames a tabletop without knowing it is one.
|
|
131
|
+
_EYE_DIAGONALS = 1.1
|
|
132
|
+
|
|
133
|
+
# How the Overview's height is split between the camera row, the scene and one
|
|
134
|
+
# text panel. The scene is what a replay is of, so it takes the most of the three.
|
|
135
|
+
# The last two split the width between that column and the plots beside it.
|
|
136
|
+
_CAMERA_SHARE = 2.5
|
|
137
|
+
_SCENE_SHARE = 3.0
|
|
138
|
+
_TEXT_SHARE = 1.2
|
|
139
|
+
_SCENE_COLUMN_SHARE = 1.4
|
|
140
|
+
_PLOT_COLUMN_SHARE = 1.0
|
|
141
|
+
|
|
142
|
+
# How many of the overview group's plots go side by side before the next row,
|
|
143
|
+
# where the group numbered none of its own. A vector is three components, so a
|
|
144
|
+
# row holds one.
|
|
145
|
+
_PLOTS_PER_ROW = 3
|
|
146
|
+
|
|
147
|
+
# How much taller a row of plots is per plot beyond the first. Plots on one row
|
|
148
|
+
# split its width, so a row of three is drawn narrow and is given the height back
|
|
149
|
+
# to keep a curve legible; a row of one already has the width.
|
|
150
|
+
_ROW_SHARE_PER_PLOT = 0.25
|
|
151
|
+
|
|
152
|
+
# Keep the flush timeout below the process deadline, so that a flush expiring
|
|
153
|
+
# while the parent still waits raises in `_build_recording` and reaches it as a
|
|
154
|
+
# message rather than as a killed child. Only a flush starting within the
|
|
155
|
+
# difference between the two bounds expires first: the process deadline is what
|
|
156
|
+
# ends a build whose later flush stalls.
|
|
157
|
+
RERUN_FLUSH_TIMEOUT = timedelta(minutes=4)
|
|
158
|
+
RECORDING_TIMEOUT = timedelta(minutes=5)
|
|
159
|
+
RECORDING_KILL_TIMEOUT = timedelta(seconds=10)
|
|
160
|
+
|
|
161
|
+
# How many recordings one runner process builds at once, so two runners on one
|
|
162
|
+
# box build eight. A build takes one CPU core for x264 or one NVENC session per
|
|
163
|
+
# camera, shared with the benchmark, and a GeForce card opens eight NVENC sessions.
|
|
164
|
+
MAX_CONCURRENT_RECORDINGS = 4
|
|
165
|
+
_RECORDING_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RECORDINGS)
|
|
166
|
+
|
|
167
|
+
# `Process.start` reaps every finished `multiprocessing` child, then stores its
|
|
168
|
+
# exit code. Starting a child, and joining it to read its exit code, both hold
|
|
169
|
+
# this lock, so a read never runs between the reap and the store.
|
|
170
|
+
_REAP_LOCK = threading.Lock()
|
|
171
|
+
|
|
172
|
+
# Maximum pose difference for a body to count as still.
|
|
173
|
+
_STILL_METRES = 1e-6
|
|
174
|
+
|
|
175
|
+
# Minimum body-to-moving-scene diagonal ratio for scenery.
|
|
176
|
+
_SCENERY_DIAGONALS = 3.0
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def replay_logs_in(results_dir: Path) -> list[Path]:
|
|
180
|
+
"""Every replay log a benchmark worker left in `results_dir`.
|
|
181
|
+
|
|
182
|
+
Empty is the ordinary answer: a benchmark that writes no logs still runs, and
|
|
183
|
+
its episodes get camera and plot views from nothing at all (ADR 0003).
|
|
184
|
+
"""
|
|
185
|
+
return sorted(results_dir.glob(f"*{REPLAY_LOG_SUFFIX}"))
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _event_rows(line: str) -> list[tuple[str, str]]:
|
|
189
|
+
"""One channel value as the rows it is worth: each body, and the level it declared.
|
|
190
|
+
|
|
191
|
+
A tag that is read is REMOVED from the body, because a level rendered in its own
|
|
192
|
+
column and repeated at the head of the sentence is the same fact written twice.
|
|
193
|
+
|
|
194
|
+
A row that declared no level gets `_DEFAULT_LEVEL` rather than nothing, so it is
|
|
195
|
+
written rather than left to inherit.
|
|
196
|
+
"""
|
|
197
|
+
rows: list[tuple[str, str]] = []
|
|
198
|
+
for piece in (p.strip() for p in line.split(_EVENT_SEPARATOR)):
|
|
199
|
+
if not piece:
|
|
200
|
+
continue
|
|
201
|
+
tag = _LEVEL_TAG.match(piece)
|
|
202
|
+
if tag is None:
|
|
203
|
+
rows.append((piece, _DEFAULT_LEVEL))
|
|
204
|
+
else:
|
|
205
|
+
rows.append((piece[tag.end() :], tag.group(1)))
|
|
206
|
+
return rows
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class RecordingError(Exception):
|
|
210
|
+
"""A recording could not be built.
|
|
211
|
+
|
|
212
|
+
Carries what the child process reported, or how it died where it reported
|
|
213
|
+
nothing -- a segfault or an OOM kill leaves an exit code and no message.
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class RecordingTimeoutError(RecordingError):
|
|
218
|
+
"""A recording build did not finish within `RECORDING_TIMEOUT`.
|
|
219
|
+
|
|
220
|
+
Separate from its base so that a caller can stop rebuilding the log: a build
|
|
221
|
+
that expired takes the whole deadline again on the next attempt, where a
|
|
222
|
+
build that failed may succeed.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class RecordingWriter:
|
|
227
|
+
"""Builds an episode's recording in a child process.
|
|
228
|
+
|
|
229
|
+
The work happens in `_build_recording` in a spawned child rather than here:
|
|
230
|
+
see this module's docstring for why a recording stream cannot be built in a
|
|
231
|
+
long-lived process. The parent reads none of the log, so an episode's frames
|
|
232
|
+
never enter the daemon's own memory.
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
def write_recording(
|
|
236
|
+
self, log_path: Path, output_path: Path, *, run_id: UUID
|
|
237
|
+
) -> None:
|
|
238
|
+
"""Build the recording for the log at `log_path`, at `output_path`.
|
|
239
|
+
|
|
240
|
+
Takes the log's path rather than a `ReplayLog`: the child reads it, so
|
|
241
|
+
the frames are decoded once, in the process that is about to exit. Handing
|
|
242
|
+
it a parsed log would mean decoding an episode here and pickling it across
|
|
243
|
+
a pipe, which is the cost this exists to avoid.
|
|
244
|
+
|
|
245
|
+
Waits while `MAX_CONCURRENT_RECORDINGS` builds are running. The deadline
|
|
246
|
+
starts when the child does.
|
|
247
|
+
|
|
248
|
+
Raises:
|
|
249
|
+
RecordingTimeoutError: The child did not finish within
|
|
250
|
+
`RECORDING_TIMEOUT`, or stayed alive after being killed.
|
|
251
|
+
RecordingError: The child failed, or died without reporting why.
|
|
252
|
+
"""
|
|
253
|
+
with _RECORDING_SLOTS:
|
|
254
|
+
self._write_in_child(log_path, output_path, run_id=run_id)
|
|
255
|
+
|
|
256
|
+
def _write_in_child(
|
|
257
|
+
self, log_path: Path, output_path: Path, *, run_id: UUID
|
|
258
|
+
) -> None:
|
|
259
|
+
context = mp.get_context("spawn")
|
|
260
|
+
# spawn, not fork: the daemon holds an event loop, worker threads and
|
|
261
|
+
# sockets that a forked child would inherit half-owned.
|
|
262
|
+
reported = context.SimpleQueue()
|
|
263
|
+
child = context.Process(
|
|
264
|
+
target=_build_recording,
|
|
265
|
+
args=(str(log_path), str(output_path), run_id.hex, reported),
|
|
266
|
+
)
|
|
267
|
+
with _REAP_LOCK:
|
|
268
|
+
child.start()
|
|
269
|
+
timed_out = not connection.wait(
|
|
270
|
+
[child.sentinel], RECORDING_TIMEOUT.total_seconds()
|
|
271
|
+
)
|
|
272
|
+
if timed_out:
|
|
273
|
+
child.kill()
|
|
274
|
+
# Raise before reading `reported`: a child killed part of the way
|
|
275
|
+
# through `put` leaves the first bytes of a message in the pipe, and
|
|
276
|
+
# `get` waits for the rest of it without a deadline.
|
|
277
|
+
if not connection.wait(
|
|
278
|
+
[child.sentinel], RECORDING_KILL_TIMEOUT.total_seconds()
|
|
279
|
+
):
|
|
280
|
+
raise RecordingTimeoutError(
|
|
281
|
+
f"the recording process for {log_path.name} did not stop "
|
|
282
|
+
f"within {RECORDING_KILL_TIMEOUT.total_seconds():g}s after "
|
|
283
|
+
"being killed"
|
|
284
|
+
)
|
|
285
|
+
# Read on the deadline path too: a child that reported its failure and
|
|
286
|
+
# then stalled on the way out is what the deadline catches, and the
|
|
287
|
+
# message is the only thing separating that from a build that was slow.
|
|
288
|
+
failure = None if reported.empty() else reported.get()
|
|
289
|
+
if timed_out:
|
|
290
|
+
said = "" if failure is None else f", after reporting {failure}"
|
|
291
|
+
raise RecordingTimeoutError(
|
|
292
|
+
f"the recording process for {log_path.name} exceeded "
|
|
293
|
+
f"{RECORDING_TIMEOUT.total_seconds():g}s{said}"
|
|
294
|
+
)
|
|
295
|
+
if failure is not None:
|
|
296
|
+
raise RecordingError(str(failure))
|
|
297
|
+
with _REAP_LOCK:
|
|
298
|
+
child.join()
|
|
299
|
+
exitcode = child.exitcode
|
|
300
|
+
if exitcode != 0:
|
|
301
|
+
raise RecordingError(
|
|
302
|
+
f"the recording process for {log_path.name} exited {exitcode}"
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
def render(self, log: ReplayLog, output_path: Path, *, run_id: UUID) -> None:
|
|
306
|
+
"""Write `log` as the episode's recording at `output_path`, in this process.
|
|
307
|
+
|
|
308
|
+
`run_id` is in the store id rather than only the filename. Rerun merges two
|
|
309
|
+
recordings sharing one, and an episode index alone is shared by every run, so
|
|
310
|
+
a viewer handed episode 4 of two runs would interleave them onto one timeline.
|
|
311
|
+
|
|
312
|
+
The blueprint is flushed before the episode is logged, so it is written
|
|
313
|
+
ahead of it. A viewer applies a blueprint where it meets it in the stream
|
|
314
|
+
and draws its own default layout until then, so a layout that lands after
|
|
315
|
+
the frames is a layout the viewer never applies to them. Logging it first
|
|
316
|
+
is not enough on its own: the chunk batcher decides what reaches the sink
|
|
317
|
+
and when, and under load it has written the episode first.
|
|
318
|
+
|
|
319
|
+
NOTE: this is where the leak in this module's docstring lives. The stream
|
|
320
|
+
it opens cannot be closed, so every call costs the calling process two
|
|
321
|
+
threads and their buffers, permanently. Only a process about to exit may
|
|
322
|
+
call it, which here is a test. Everything else wants `write_recording`.
|
|
323
|
+
|
|
324
|
+
Raises:
|
|
325
|
+
RuntimeError: A flush did not finish within `RERUN_FLUSH_TIMEOUT`.
|
|
326
|
+
"""
|
|
327
|
+
self._render(
|
|
328
|
+
log,
|
|
329
|
+
replace(_survey(log.steps), depth_ranges=depth_ranges(log)),
|
|
330
|
+
log.steps,
|
|
331
|
+
output_path,
|
|
332
|
+
run_id=run_id,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
def render_file(self, log_path: Path, output_path: Path, *, run_id: UUID) -> None:
|
|
336
|
+
"""Write the log at `log_path` as its recording, reading one step at a time.
|
|
337
|
+
|
|
338
|
+
Two cheap passes come first: one reads the poses alone, without decoding a
|
|
339
|
+
frame, for where the 3D view's camera starts, and one decodes the few steps
|
|
340
|
+
the depth colour range samples. The third pass encodes and logs each step
|
|
341
|
+
and drops it before reading the next, so the recording file is the only
|
|
342
|
+
thing that grows with the episode.
|
|
343
|
+
|
|
344
|
+
The same NOTE as `render` applies: only a process about to exit may call it,
|
|
345
|
+
which is `_build_recording`.
|
|
346
|
+
"""
|
|
347
|
+
head = read_replay_head(log_path)
|
|
348
|
+
survey = _survey(iter_replay_steps(log_path, decode=lambda _: False))
|
|
349
|
+
stride = _range_stride(survey.steps)
|
|
350
|
+
sampled = iter_replay_steps(log_path, decode=lambda index: index % stride == 0)
|
|
351
|
+
survey = replace(survey, depth_ranges=_depth_ranges(sampled))
|
|
352
|
+
self._render(
|
|
353
|
+
head,
|
|
354
|
+
survey,
|
|
355
|
+
iter_replay_steps(log_path),
|
|
356
|
+
output_path,
|
|
357
|
+
run_id=run_id,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def _render(
|
|
361
|
+
self,
|
|
362
|
+
head: ReplayLog,
|
|
363
|
+
survey: "_Survey",
|
|
364
|
+
steps: Iterable[ReplayStep],
|
|
365
|
+
output_path: Path,
|
|
366
|
+
*,
|
|
367
|
+
run_id: UUID,
|
|
368
|
+
) -> None:
|
|
369
|
+
stream = rr.RecordingStream(
|
|
370
|
+
"manifold",
|
|
371
|
+
recording_id=f"{run_id.hex}-episode-{head.episode_idx}",
|
|
372
|
+
send_properties=False,
|
|
373
|
+
)
|
|
374
|
+
stream.save(str(output_path))
|
|
375
|
+
stream.send_blueprint(blueprint_for(head, survey))
|
|
376
|
+
# Ahead of the episode rather than merely before it in call order:
|
|
377
|
+
# see `render` on why the batcher needs telling.
|
|
378
|
+
stream.flush(timeout_sec=RERUN_FLUSH_TIMEOUT.total_seconds())
|
|
379
|
+
self._log_cameras(stream, head)
|
|
380
|
+
self._log_scene(stream, head)
|
|
381
|
+
self._log_steps(stream, head, survey, steps)
|
|
382
|
+
# Blocking, and before the caller moves on: the sink thread writes the
|
|
383
|
+
# file, and a process that exited without waiting for it would leave
|
|
384
|
+
# the recording truncated.
|
|
385
|
+
stream.flush(timeout_sec=RERUN_FLUSH_TIMEOUT.total_seconds())
|
|
386
|
+
|
|
387
|
+
def _log_cameras(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
|
|
388
|
+
"""Log each camera's pinhole once, so its depth frame can be projected.
|
|
389
|
+
|
|
390
|
+
Static, because intrinsics are: the header declares them for the run and the
|
|
391
|
+
per-step half of a calibration is the pose (ADR 0008). Without this a depth
|
|
392
|
+
frame is a 2D image the viewer cannot place in the scene; with it, and with
|
|
393
|
+
the per-step `Transform3D` below, Rerun back-projects it against the bodies.
|
|
394
|
+
|
|
395
|
+
`image_from_camera` is the 3x3 the intrinsics imply. Rerun reads a pinhole in
|
|
396
|
+
the same axes the log fixes for the pose — OpenCV, x right, y down, z forward
|
|
397
|
+
— which is `RDF`, so it is stated rather than left to the default.
|
|
398
|
+
"""
|
|
399
|
+
for camera in log.cameras:
|
|
400
|
+
stream.log(
|
|
401
|
+
f"/{_CAMERAS_ROOT}/{camera.name}",
|
|
402
|
+
rr.Pinhole(
|
|
403
|
+
image_from_camera=[
|
|
404
|
+
[camera.fx, 0.0, camera.cx],
|
|
405
|
+
[0.0, camera.fy, camera.cy],
|
|
406
|
+
[0.0, 0.0, 1.0],
|
|
407
|
+
],
|
|
408
|
+
width=camera.width,
|
|
409
|
+
height=camera.height,
|
|
410
|
+
camera_xyz=rr.ViewCoordinates.RDF,
|
|
411
|
+
),
|
|
412
|
+
static=True,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
def _log_samples(
|
|
416
|
+
self,
|
|
417
|
+
stream: rr.RecordingStream,
|
|
418
|
+
path: str,
|
|
419
|
+
samples: list[tuple[int, bytes]],
|
|
420
|
+
events: int,
|
|
421
|
+
) -> None:
|
|
422
|
+
"""Log each sample at the step of the frame it holds, on `step` alone.
|
|
423
|
+
|
|
424
|
+
An encoder can hand a sample back a few frames after its own, so each is
|
|
425
|
+
stamped with its frame's step, and the step being logged is set again by
|
|
426
|
+
the caller's next step. The `event` timeline is left off the samples and
|
|
427
|
+
restored after them, so the rows that follow keep the event they had.
|
|
428
|
+
"""
|
|
429
|
+
if not samples:
|
|
430
|
+
return
|
|
431
|
+
stream.disable_timeline(_EVENT_TIMELINE)
|
|
432
|
+
for step, sample in samples:
|
|
433
|
+
stream.set_time(_STEP_TIMELINE, sequence=step)
|
|
434
|
+
stream.log(path, rr.VideoStream.from_fields(sample=sample))
|
|
435
|
+
if events > 0:
|
|
436
|
+
stream.set_time(_EVENT_TIMELINE, sequence=events - 1)
|
|
437
|
+
|
|
438
|
+
def _log_scene(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
|
|
439
|
+
"""Log every body's geometry once, as static data under its own entity.
|
|
440
|
+
|
|
441
|
+
A part's pose is static and its body's is per step, and Rerun composes a
|
|
442
|
+
transform down the entity path, so the parts follow the body without being
|
|
443
|
+
written again.
|
|
444
|
+
"""
|
|
445
|
+
for body in log.bodies:
|
|
446
|
+
for index, part in enumerate(body.parts):
|
|
447
|
+
path = f"{_body_path(body.name)}/{index}"
|
|
448
|
+
stream.log(
|
|
449
|
+
path,
|
|
450
|
+
rr.Transform3D(
|
|
451
|
+
translation=part.position.tolist(),
|
|
452
|
+
quaternion=rr.Quaternion(xyzw=part.orientation),
|
|
453
|
+
),
|
|
454
|
+
static=True,
|
|
455
|
+
)
|
|
456
|
+
vertices = part.mesh.vertices.astype(np.float32)
|
|
457
|
+
faces = part.mesh.faces.astype(np.uint32)
|
|
458
|
+
stream.log(
|
|
459
|
+
path,
|
|
460
|
+
rr.Mesh3D(
|
|
461
|
+
vertex_positions=vertices,
|
|
462
|
+
triangle_indices=faces,
|
|
463
|
+
vertex_normals=_vertex_normals(vertices, faces),
|
|
464
|
+
albedo_factor=[round(channel * 255) for channel in part.color],
|
|
465
|
+
),
|
|
466
|
+
static=True,
|
|
467
|
+
)
|
|
468
|
+
self._log_scene_labels(stream, log)
|
|
469
|
+
|
|
470
|
+
def _log_scene_labels(self, stream: rr.RecordingStream, log: ReplayLog) -> None:
|
|
471
|
+
"""Log the body labels, the fingertip pads and the step length as static JSON.
|
|
472
|
+
|
|
473
|
+
Each body and pad is keyed by the entity path of its pose, so a reader matches
|
|
474
|
+
it to the poses without knowing how a body name becomes a path. A log that
|
|
475
|
+
labels nothing logs nothing.
|
|
476
|
+
"""
|
|
477
|
+
labels = {
|
|
478
|
+
_body_path(body.name): {"role": body.role, "object": body.object}
|
|
479
|
+
for body in log.bodies
|
|
480
|
+
if body.role is not None
|
|
481
|
+
}
|
|
482
|
+
if not labels and not log.finger_pads:
|
|
483
|
+
return
|
|
484
|
+
scene = {
|
|
485
|
+
"bodies": labels,
|
|
486
|
+
"finger_pads": [
|
|
487
|
+
{"body": _body_path(pad.body), "offset": pad.offset.tolist()}
|
|
488
|
+
for pad in log.finger_pads
|
|
489
|
+
],
|
|
490
|
+
"step_seconds": log.step_seconds,
|
|
491
|
+
}
|
|
492
|
+
stream.log(
|
|
493
|
+
f"/{_MANIFOLD_SCENE_ENTITY}",
|
|
494
|
+
rr.AnyValues(scene=json.dumps(scene)),
|
|
495
|
+
static=True,
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
def _log_steps(
|
|
499
|
+
self,
|
|
500
|
+
stream: rr.RecordingStream,
|
|
501
|
+
log: ReplayLog,
|
|
502
|
+
survey: "_Survey",
|
|
503
|
+
steps: Iterable[ReplayStep],
|
|
504
|
+
) -> None:
|
|
505
|
+
"""Log each step's body poses and channel values on the `step` timeline.
|
|
506
|
+
|
|
507
|
+
A body that never moves is logged once, as a static pose, rather than on every
|
|
508
|
+
step.
|
|
509
|
+
|
|
510
|
+
A depth frame is logged as the array the reader decoded. A colour frame is
|
|
511
|
+
sent to its camera's H.264 encoder, one per camera, opened at the camera's
|
|
512
|
+
first frame, and the samples it finishes are logged at once. The codec is
|
|
513
|
+
static on the camera's entity.
|
|
514
|
+
|
|
515
|
+
A text channel is logged only where its line changed. Rerun renders text as a
|
|
516
|
+
log, one row per entry, so a channel carrying the same line every step - a task
|
|
517
|
+
instruction, say - gives a panel of one sentence repeated as many times as the
|
|
518
|
+
episode is long, and anything else the benchmark said is lost among the copies.
|
|
519
|
+
|
|
520
|
+
A changed line is then split into the rows it is worth, each stamped on the
|
|
521
|
+
`event` timeline as well as `step` and carrying whatever level it declared.
|
|
522
|
+
The dedup above still compares the JOINED value, because what it exists to
|
|
523
|
+
suppress is a channel repeating itself, and splitting must not change which
|
|
524
|
+
steps it treats as having spoken.
|
|
525
|
+
"""
|
|
526
|
+
# Each plot is one entity, and each channel one series of it.
|
|
527
|
+
members: dict[str, list[str]] = {}
|
|
528
|
+
slots: dict[str, tuple[str, int]] = {}
|
|
529
|
+
for channel in log.channels:
|
|
530
|
+
if channel.kind is not ChannelKind.SCALAR:
|
|
531
|
+
continue
|
|
532
|
+
origin = _scalar_view(channel)
|
|
533
|
+
names = members.setdefault(origin, [])
|
|
534
|
+
slots[channel.name] = (origin, len(names))
|
|
535
|
+
leaf = channel.name.rsplit("/", 1)[-1]
|
|
536
|
+
names.append(f"{origin}/{leaf}" if "/" in channel.name else origin)
|
|
537
|
+
series = {
|
|
538
|
+
origin: [_series_name(path, members) for path in paths]
|
|
539
|
+
for origin, paths in members.items()
|
|
540
|
+
}
|
|
541
|
+
said: dict[str, str] = {}
|
|
542
|
+
# Monotonic across the episode, so no event hides behind another.
|
|
543
|
+
events = 0
|
|
544
|
+
# The last camera pose written, so an unmoved camera is not written again.
|
|
545
|
+
placed: dict[str, tuple[bytes, bytes]] = {}
|
|
546
|
+
# Once for the episode, ahead of the frames it applies to.
|
|
547
|
+
ranges = survey.depth_ranges
|
|
548
|
+
videos: dict[str, CameraVideo] = {}
|
|
549
|
+
poses: dict[str, _Column] = {}
|
|
550
|
+
scalars: dict[str, _Column] = {}
|
|
551
|
+
still = set(survey.first_poses) - survey.moving
|
|
552
|
+
for name in still:
|
|
553
|
+
pose = survey.first_poses[name]
|
|
554
|
+
stream.log(
|
|
555
|
+
_body_path(name),
|
|
556
|
+
rr.Transform3D(
|
|
557
|
+
translation=pose.position.tolist(),
|
|
558
|
+
quaternion=rr.Quaternion(xyzw=pose.orientation),
|
|
559
|
+
),
|
|
560
|
+
static=True,
|
|
561
|
+
)
|
|
562
|
+
for step in steps:
|
|
563
|
+
stream.set_time(_STEP_TIMELINE, sequence=step.index)
|
|
564
|
+
event = events - 1 if events > 0 else None
|
|
565
|
+
for name, pose in step.poses.items():
|
|
566
|
+
if name in still:
|
|
567
|
+
continue
|
|
568
|
+
column = poses.setdefault(_body_path(name), _Column())
|
|
569
|
+
column.add(step.index, event, (pose.position, pose.orientation))
|
|
570
|
+
# A camera's pose belongs on the same entity as its frames, so the
|
|
571
|
+
# transform composes down to the image the way a body's composes down to
|
|
572
|
+
# its parts.
|
|
573
|
+
#
|
|
574
|
+
# Logged only where it changed, like a text channel below: the reader
|
|
575
|
+
# hands us every camera on every step (it fills a log's sparse
|
|
576
|
+
# extrinsics forward), and writing all of them would put the rows back
|
|
577
|
+
# that the log went to the trouble of not storing. Rerun holds a
|
|
578
|
+
# transform until the next one, so a bolted-down camera keeps its pose
|
|
579
|
+
# for the whole episode off a single row.
|
|
580
|
+
for name, pose in step.extrinsics.items():
|
|
581
|
+
seen = placed.get(name)
|
|
582
|
+
current = (pose.position.tobytes(), pose.orientation.tobytes())
|
|
583
|
+
if seen == current:
|
|
584
|
+
continue
|
|
585
|
+
placed[name] = current
|
|
586
|
+
stream.log(
|
|
587
|
+
f"/{_CAMERAS_ROOT}/{name}",
|
|
588
|
+
rr.Transform3D(
|
|
589
|
+
translation=pose.position.tolist(),
|
|
590
|
+
quaternion=rr.Quaternion(xyzw=pose.orientation),
|
|
591
|
+
),
|
|
592
|
+
)
|
|
593
|
+
for name, image in step.images.items():
|
|
594
|
+
if image.dtype != np.uint8:
|
|
595
|
+
stream.log(
|
|
596
|
+
f"/{_CAMERAS_ROOT}/{name}",
|
|
597
|
+
self._as_depth_image(image, ranges.get(name)),
|
|
598
|
+
)
|
|
599
|
+
rows: dict[str, np.ndarray] = {}
|
|
600
|
+
for name, value in step.scalars.items():
|
|
601
|
+
if name not in slots:
|
|
602
|
+
raise ValueError(f"scalar channel {name!r} is not in the header")
|
|
603
|
+
origin, slot = slots[name]
|
|
604
|
+
row = rows.get(origin)
|
|
605
|
+
if row is None:
|
|
606
|
+
row = rows[origin] = np.full(len(series[origin]), np.nan)
|
|
607
|
+
row[slot] = value
|
|
608
|
+
for origin, row in rows.items():
|
|
609
|
+
scalars.setdefault(origin, _Column()).add(step.index, event, row)
|
|
610
|
+
# === TODO: Currently overfitting to RoboLab ====================
|
|
611
|
+
#
|
|
612
|
+
# Every TEXT channel goes through the event splitter, including one
|
|
613
|
+
# carrying a single sentence rather than a list of events. An
|
|
614
|
+
# `instruction` that contains " / " is split into rows that are not
|
|
615
|
+
# events, and one opening with a bracketed level loses it from the body.
|
|
616
|
+
# Latent rather than live: no `instruction` in any recorded run does
|
|
617
|
+
# either. Only a channel that declares itself a stream of events should be
|
|
618
|
+
# split; the rest should be logged whole, which is what they were before.
|
|
619
|
+
for name, line in step.text.items():
|
|
620
|
+
if said.get(name) == line:
|
|
621
|
+
continue
|
|
622
|
+
said[name] = line
|
|
623
|
+
for body, level in _event_rows(line):
|
|
624
|
+
stream.set_time(_EVENT_TIMELINE, sequence=events)
|
|
625
|
+
events += 1
|
|
626
|
+
stream.log(
|
|
627
|
+
f"/{_LOGS_ROOT}/{name}",
|
|
628
|
+
rr.TextLog(body, level=_LEVELS[level]),
|
|
629
|
+
)
|
|
630
|
+
# The Manifold events, as JSON on their own entity: grading reads them,
|
|
631
|
+
# and keeping them out of the logs keeps them out of the text panel.
|
|
632
|
+
if step.events:
|
|
633
|
+
stream.log(
|
|
634
|
+
f"/{_MANIFOLD_EVENTS_ENTITY}",
|
|
635
|
+
rr.AnyValues(events=json.dumps(list(step.events))),
|
|
636
|
+
)
|
|
637
|
+
# Last in the step: a sample moves the `step` timeline to its own frame.
|
|
638
|
+
for name, image in step.images.items():
|
|
639
|
+
if image.dtype != np.uint8:
|
|
640
|
+
continue
|
|
641
|
+
path = f"/{_CAMERAS_ROOT}/{name}"
|
|
642
|
+
video = videos.get(name)
|
|
643
|
+
if video is None:
|
|
644
|
+
stream.log(
|
|
645
|
+
path, rr.VideoStream(codec=rr.VideoCodec.H264), static=True
|
|
646
|
+
)
|
|
647
|
+
video = videos[name] = CameraVideo(name)
|
|
648
|
+
self._log_samples(stream, path, video.encode(step.index, image), events)
|
|
649
|
+
for name, video in videos.items():
|
|
650
|
+
self._log_samples(
|
|
651
|
+
stream, f"/{_CAMERAS_ROOT}/{name}", video.finish(), events
|
|
652
|
+
)
|
|
653
|
+
for path, column in poses.items():
|
|
654
|
+
for indexes, values in column.parts():
|
|
655
|
+
stream.send_columns(
|
|
656
|
+
path,
|
|
657
|
+
indexes=indexes,
|
|
658
|
+
columns=rr.Transform3D.columns(
|
|
659
|
+
translation=np.stack([position for position, _ in values]),
|
|
660
|
+
quaternion=np.stack([orientation for _, orientation in values]),
|
|
661
|
+
),
|
|
662
|
+
)
|
|
663
|
+
for path, column in scalars.items():
|
|
664
|
+
stream.log(path, rr.SeriesLines(names=series[path]), static=True)
|
|
665
|
+
for indexes, values in column.parts():
|
|
666
|
+
stacked = np.stack(values).astype(np.float64)
|
|
667
|
+
stream.send_columns(
|
|
668
|
+
path,
|
|
669
|
+
indexes=indexes,
|
|
670
|
+
columns=rr.Scalars.columns(scalars=stacked.reshape(-1)).partition(
|
|
671
|
+
[stacked.shape[1]] * stacked.shape[0]
|
|
672
|
+
),
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
def _as_depth_image(
|
|
676
|
+
self, array: np.ndarray, depth_range: tuple[float, float] | None = None
|
|
677
|
+
) -> rr.DepthImage:
|
|
678
|
+
"""A depth frame, in floating-point metres as the log carries it.
|
|
679
|
+
|
|
680
|
+
`rr.DepthImage` stores the array's dtype as its format.
|
|
681
|
+
|
|
682
|
+
`depth_range` fixes the colour map for the whole episode. Without it the
|
|
683
|
+
viewer ranges each frame on its own min and max, and a wrist camera's far
|
|
684
|
+
bound moves as the gripper does — measured at 3.2x between the widest and
|
|
685
|
+
narrowest frame of one episode, and 163% in a single step. The effect is a
|
|
686
|
+
depth view whose colours change fast while the geometry barely moves, which
|
|
687
|
+
reads as the depth itself being unstable.
|
|
688
|
+
"""
|
|
689
|
+
return rr.DepthImage(array, meter=1.0, depth_range=depth_range)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
# Pixels sampled per depth frame, and the most frames sampled, when fixing a colour
|
|
693
|
+
# range. A percentile does not need every pixel of every step: half a percent of a few
|
|
694
|
+
# hundred thousand samples is already stable well inside a millimetre, and a 520-step
|
|
695
|
+
# episode's depth is tens of millions of values.
|
|
696
|
+
_RANGE_SAMPLE_STRIDE = 7
|
|
697
|
+
_RANGE_MAX_FRAMES = 64
|
|
698
|
+
|
|
699
|
+
# The percentiles the fixed colour range spans, chosen by measuring the alternatives
|
|
700
|
+
# rather than by argument. See `depth_ranges`.
|
|
701
|
+
_RANGE_PERCENTILES = (0.5, 99.5)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _build_recording(
|
|
705
|
+
log_path: str, output_path: str, run_id_hex: str, reported: Any
|
|
706
|
+
) -> None:
|
|
707
|
+
"""Read one log and write its recording. Runs in a child process.
|
|
708
|
+
|
|
709
|
+
Module level and taking only strings, because `spawn` pickles the target and
|
|
710
|
+
its arguments to reach a fresh interpreter.
|
|
711
|
+
|
|
712
|
+
A failure is put on `reported` rather than raised: a traceback in a child
|
|
713
|
+
reaches the parent's journal as a bare exit code otherwise, and the caller
|
|
714
|
+
keeps a log eligible on the strength of the message.
|
|
715
|
+
"""
|
|
716
|
+
try:
|
|
717
|
+
RecordingWriter().render_file(
|
|
718
|
+
Path(log_path),
|
|
719
|
+
Path(output_path),
|
|
720
|
+
run_id=UUID(run_id_hex),
|
|
721
|
+
)
|
|
722
|
+
# Broad: this is the child's one channel to the parent, and an exception it
|
|
723
|
+
# let escape would reach the journal as a bare exit code.
|
|
724
|
+
except Exception as exc:
|
|
725
|
+
reported.put(f"{type(exc).__name__}: {exc}")
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
@dataclass
|
|
729
|
+
class _Column:
|
|
730
|
+
"""One entity's rows, each with its step and the event in effect at that step."""
|
|
731
|
+
|
|
732
|
+
steps: list[int] = field(default_factory=list)
|
|
733
|
+
events: list[int] = field(default_factory=list)
|
|
734
|
+
values: list[Any] = field(default_factory=list)
|
|
735
|
+
|
|
736
|
+
def add(self, step: int, event: int | None, value: Any) -> None:
|
|
737
|
+
self.steps.append(step)
|
|
738
|
+
self.values.append(value)
|
|
739
|
+
if event is not None:
|
|
740
|
+
self.events.append(event)
|
|
741
|
+
|
|
742
|
+
def parts(self) -> list[tuple[list[rr.TimeColumn], list[Any]]]:
|
|
743
|
+
"""The rows as time columns and values, split where the event time starts.
|
|
744
|
+
|
|
745
|
+
A row logged before the first event carries no event time, and a column
|
|
746
|
+
holds a time for every row or for none.
|
|
747
|
+
"""
|
|
748
|
+
split = len(self.steps) - len(self.events)
|
|
749
|
+
parts = []
|
|
750
|
+
if split > 0:
|
|
751
|
+
steps = rr.TimeColumn(_STEP_TIMELINE, sequence=self.steps[:split])
|
|
752
|
+
parts.append(([steps], self.values[:split]))
|
|
753
|
+
if self.events:
|
|
754
|
+
steps = rr.TimeColumn(_STEP_TIMELINE, sequence=self.steps[split:])
|
|
755
|
+
events = rr.TimeColumn(_EVENT_TIMELINE, sequence=self.events)
|
|
756
|
+
parts.append(([steps, events], self.values[split:]))
|
|
757
|
+
return parts
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
@dataclass
|
|
761
|
+
class _Survey:
|
|
762
|
+
"""What the layout and the depth colour map need from the whole episode.
|
|
763
|
+
|
|
764
|
+
Small whatever the episode's length: the first step's poses, the names of the
|
|
765
|
+
bodies that move, the step count and one range per depth channel.
|
|
766
|
+
"""
|
|
767
|
+
|
|
768
|
+
first_poses: dict[str, Pose]
|
|
769
|
+
moving: set[str]
|
|
770
|
+
steps: int
|
|
771
|
+
depth_ranges: dict[str, tuple[float, float]] = field(default_factory=dict)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _survey(steps: Iterable[ReplayStep]) -> _Survey:
|
|
775
|
+
"""Walk the steps once for the first poses, the bodies that move and the count."""
|
|
776
|
+
first: dict[str, Pose] | None = None
|
|
777
|
+
moving: set[str] = set()
|
|
778
|
+
count = 0
|
|
779
|
+
for step in steps:
|
|
780
|
+
count += 1
|
|
781
|
+
if first is None:
|
|
782
|
+
first = dict(step.poses)
|
|
783
|
+
continue
|
|
784
|
+
moving.update(
|
|
785
|
+
name
|
|
786
|
+
for name, pose in step.poses.items()
|
|
787
|
+
if name in first
|
|
788
|
+
and name not in moving
|
|
789
|
+
and not _same_pose(pose, first[name])
|
|
790
|
+
)
|
|
791
|
+
return _Survey(first_poses=first or {}, moving=moving, steps=count)
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def _range_stride(steps: int) -> int:
|
|
795
|
+
return max(1, -(-steps // _RANGE_MAX_FRAMES))
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def depth_ranges(log: ReplayLog) -> dict[str, tuple[float, float]]:
|
|
799
|
+
"""One fixed colour range per depth channel, spanning the 0.5-99.5 percentiles.
|
|
800
|
+
|
|
801
|
+
Fixed for the episode on purpose: a range recomputed per frame is what makes a
|
|
802
|
+
depth view's colours move while its geometry does not (see `_as_depth_image`).
|
|
803
|
+
|
|
804
|
+
Percentiles rather than the extremes or a multiple of sigma, because the choice
|
|
805
|
+
was measured. Over the first two episodes of all four suites of the 80-episode
|
|
806
|
+
sweep, counting how many of a 256-entry colour ramp the data actually exercises
|
|
807
|
+
(the entropy of the quantisation, as 2**bits):
|
|
808
|
+
|
|
809
|
+
| rule | agentview | wrist | clipped |
|
|
810
|
+
| --------------- | --------- | ----- | ------- |
|
|
811
|
+
| min/max | 112 | 103 | 0.00% |
|
|
812
|
+
| mean +/- 3 sigma| 112 | 160 | 0.04% |
|
|
813
|
+
| 1/99 | 112 | 186 | 1.87% |
|
|
814
|
+
| **0.5/99.5** | **113** | 182 | 0.87% |
|
|
815
|
+
|
|
816
|
+
The scene camera is insensitive: its depth spans 0.68-3.07 m with no tail, so
|
|
817
|
+
every rule lands within one level of every other. The WRIST camera is where the
|
|
818
|
+
range is won or lost. A handful of rays slip past the gripper into the far scene
|
|
819
|
+
and stretch its extremes to 1.01 m when the surfaces that matter sit inside
|
|
820
|
+
0.36 m, so min/max spends two thirds of the ramp on a few dozen pixels and
|
|
821
|
+
leaves 103 levels for the hand. Trimming half a percent off each end recovers
|
|
822
|
+
182.
|
|
823
|
+
|
|
824
|
+
1/99 buys four more levels on the wrist for double the clipping and one fewer on
|
|
825
|
+
the scene view, which is not a trade worth making. Nothing below 2/98 is
|
|
826
|
+
competitive.
|
|
827
|
+
|
|
828
|
+
The limit worth knowing: this trims a tail only while the tail is THINNER than
|
|
829
|
+
the half percent being cut. A camera where more than 0.5% of pixels sit far out
|
|
830
|
+
puts the 99.5th percentile inside the tail, and the range degrades gracefully to
|
|
831
|
+
the raw extremes rather than misbehaving. LIBERO's wrist tail is about two rays
|
|
832
|
+
in a thousand, well inside that.
|
|
833
|
+
|
|
834
|
+
No clamp to the observed extremes is needed, unlike the sigma rule this replaces:
|
|
835
|
+
a percentile is inside the data by construction, where `mean - 3 sigma` landed
|
|
836
|
+
below zero on every camera measured and spent part of the ramp on distances no
|
|
837
|
+
pixel can reach.
|
|
838
|
+
"""
|
|
839
|
+
# Subsample frames as well as pixels. A percentile over the episode does not need
|
|
840
|
+
# every step, and a long episode's samples would otherwise be tens of megabytes.
|
|
841
|
+
return _depth_ranges(log.steps[:: _range_stride(len(log.steps))])
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _depth_ranges(steps: Iterable[ReplayStep]) -> dict[str, tuple[float, float]]:
|
|
845
|
+
samples: dict[str, list[np.ndarray]] = {}
|
|
846
|
+
for step in steps:
|
|
847
|
+
for name, frame in step.images.items():
|
|
848
|
+
if frame.dtype == np.uint8:
|
|
849
|
+
continue
|
|
850
|
+
flat = np.asarray(frame, dtype=np.float32).reshape(-1)
|
|
851
|
+
values = flat[::_RANGE_SAMPLE_STRIDE]
|
|
852
|
+
finite = values[np.isfinite(values)]
|
|
853
|
+
if finite.size:
|
|
854
|
+
samples.setdefault(name, []).append(finite)
|
|
855
|
+
ranges: dict[str, tuple[float, float]] = {}
|
|
856
|
+
for name, chunks in samples.items():
|
|
857
|
+
low, high = (
|
|
858
|
+
float(v) for v in np.percentile(np.concatenate(chunks), _RANGE_PERCENTILES)
|
|
859
|
+
)
|
|
860
|
+
# A channel with no spread — a flat test frame, or a camera staring at one
|
|
861
|
+
# surface — collapses to a single value. Omit it rather than hand the viewer
|
|
862
|
+
# an empty range: there is no flicker to prevent on a frame that never
|
|
863
|
+
# changes, and a zero-width range is a worse instruction than none.
|
|
864
|
+
if high > low:
|
|
865
|
+
ranges[name] = (low, high)
|
|
866
|
+
return ranges
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def blueprint_for(log: ReplayLog, survey: _Survey | None = None) -> rrb.Blueprint:
|
|
870
|
+
"""The view layout this log's channels imply.
|
|
871
|
+
|
|
872
|
+
An Overview tab holds the cameras, the scene, the text and the plots of the
|
|
873
|
+
group the header named; every other scalar group becomes a tab of its own,
|
|
874
|
+
named by the group. The named group does not get a tab: its plots are already
|
|
875
|
+
on the Overview. Nothing here names a channel (ADR 0008), so a benchmark
|
|
876
|
+
chooses its own layout by naming channels, one of their groups and the rows
|
|
877
|
+
they sit on.
|
|
878
|
+
"""
|
|
879
|
+
if survey is None:
|
|
880
|
+
survey = _survey(log.steps)
|
|
881
|
+
views = _scalar_views(log)
|
|
882
|
+
overview = (
|
|
883
|
+
views.pop(log.overview_group, []) if log.overview_group is not None else []
|
|
884
|
+
)
|
|
885
|
+
tabs: list[Any] = [_overview_tab(log, survey, overview)]
|
|
886
|
+
for group, entries in views.items():
|
|
887
|
+
tabs.append(_plot_grid(entries, wrap=1, name=group))
|
|
888
|
+
return rrb.Blueprint(
|
|
889
|
+
rrb.Tabs(*tabs, active_tab=0),
|
|
890
|
+
rrb.SelectionPanel(state="collapsed"),
|
|
891
|
+
rrb.BlueprintPanel(state="hidden"),
|
|
892
|
+
rrb.TimePanel(state="collapsed"),
|
|
893
|
+
)
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _overview_tab(
|
|
897
|
+
log: ReplayLog, survey: _Survey, overview_entries: list[tuple[str, int | None]]
|
|
898
|
+
) -> Any:
|
|
899
|
+
"""The cameras over the scene and the text, beside the overview group's plots.
|
|
900
|
+
|
|
901
|
+
The cameras come first because they are what the policy saw, and the scene
|
|
902
|
+
under them is the same moment from outside it.
|
|
903
|
+
|
|
904
|
+
One column when the header named no group, which is every log that says
|
|
905
|
+
nothing about what is worth watching while the episode plays.
|
|
906
|
+
|
|
907
|
+
No scene when no body has geometry: a 3D view of poses alone draws nothing.
|
|
908
|
+
"""
|
|
909
|
+
left: list[Any] = []
|
|
910
|
+
shares: list[float] = []
|
|
911
|
+
cameras = [
|
|
912
|
+
rrb.Spatial2DView(origin=f"/{_CAMERAS_ROOT}/{channel.name}", name=channel.name)
|
|
913
|
+
for channel in log.channels
|
|
914
|
+
if channel.kind is ChannelKind.IMAGE
|
|
915
|
+
]
|
|
916
|
+
if len(cameras) > 0:
|
|
917
|
+
left.append(rrb.Horizontal(*cameras))
|
|
918
|
+
shares.append(_CAMERA_SHARE)
|
|
919
|
+
if any(len(body.parts) > 0 for body in log.bodies):
|
|
920
|
+
left.append(
|
|
921
|
+
rrb.Spatial3DView(
|
|
922
|
+
origin=f"/{_BODIES_ROOT}",
|
|
923
|
+
name="scene",
|
|
924
|
+
contents=[
|
|
925
|
+
"+ $origin/**",
|
|
926
|
+
*[
|
|
927
|
+
f"- $origin/{_entity_name(name)}/**"
|
|
928
|
+
for name in _scenery(log, survey)
|
|
929
|
+
],
|
|
930
|
+
],
|
|
931
|
+
background=rrb.Background(kind=rrb.BackgroundKind.GradientBright),
|
|
932
|
+
line_grid=rrb.LineGrid3D(spacing=0.05),
|
|
933
|
+
eye_controls=_eye_for(log, survey),
|
|
934
|
+
)
|
|
935
|
+
)
|
|
936
|
+
shares.append(_SCENE_SHARE)
|
|
937
|
+
for channel in log.channels:
|
|
938
|
+
if channel.kind is ChannelKind.TEXT:
|
|
939
|
+
left.append(
|
|
940
|
+
rrb.TextLogView(
|
|
941
|
+
origin=f"/{_LOGS_ROOT}/{channel.name}", name=channel.name
|
|
942
|
+
)
|
|
943
|
+
)
|
|
944
|
+
shares.append(_TEXT_SHARE)
|
|
945
|
+
if len(overview_entries) == 0:
|
|
946
|
+
return rrb.Vertical(*left, row_shares=shares, name="Overview")
|
|
947
|
+
return rrb.Horizontal(
|
|
948
|
+
rrb.Vertical(*left, row_shares=shares),
|
|
949
|
+
_plot_grid(overview_entries, wrap=_PLOTS_PER_ROW),
|
|
950
|
+
column_shares=[_SCENE_COLUMN_SHARE, _PLOT_COLUMN_SHARE],
|
|
951
|
+
name="Overview",
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _plot_grid(
|
|
956
|
+
entries: list[tuple[str, int | None]], *, wrap: int, name: str | None = None
|
|
957
|
+
) -> Any:
|
|
958
|
+
"""One group's plots, in the rows it numbered or in rows of `wrap`.
|
|
959
|
+
|
|
960
|
+
A row of one plot is the plot itself: it has the grid's whole width already,
|
|
961
|
+
and a container holding it alone would only be a level to walk through.
|
|
962
|
+
"""
|
|
963
|
+
rows = _plot_rows(entries, wrap=wrap)
|
|
964
|
+
return rrb.Vertical(
|
|
965
|
+
*[_plot_row(row) for row in rows],
|
|
966
|
+
row_shares=[1.0 + _ROW_SHARE_PER_PLOT * (len(row) - 1) for row in rows],
|
|
967
|
+
name=name,
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def _plot_rows(entries: list[tuple[str, int | None]], *, wrap: int) -> list[list[str]]:
|
|
972
|
+
"""The plots of one group, grouped into the rows they are drawn on.
|
|
973
|
+
|
|
974
|
+
A group numbering its rows is laid out as it asked, in ascending row number.
|
|
975
|
+
Which of its scalars belong side by side is the benchmark's to say: three
|
|
976
|
+
position axes are one vector and a gripper state is a series on its own, and a
|
|
977
|
+
channel list distinguishes neither.
|
|
978
|
+
|
|
979
|
+
A group numbering none is wrapped into rows of `wrap` instead, which is what
|
|
980
|
+
every group that says nothing about its layout gets. The writer refuses a
|
|
981
|
+
group numbering only some of its channels, so the two cases are all of them.
|
|
982
|
+
"""
|
|
983
|
+
numbered = [(origin, row) for origin, row in entries if row is not None]
|
|
984
|
+
if len(entries) > 0 and len(numbered) == len(entries):
|
|
985
|
+
rows: dict[int, list[str]] = {}
|
|
986
|
+
for origin, row in numbered:
|
|
987
|
+
rows.setdefault(row, []).append(origin)
|
|
988
|
+
return [rows[number] for number in sorted(rows)]
|
|
989
|
+
origins = [origin for origin, _ in entries]
|
|
990
|
+
return [origins[at : at + wrap] for at in range(0, len(origins), wrap)]
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
def _plot_row(origins: list[str]) -> Any:
|
|
994
|
+
"""One row of plots, side by side."""
|
|
995
|
+
views = [
|
|
996
|
+
rrb.TimeSeriesView(origin=origin, name=origin.rsplit("/", 1)[-1])
|
|
997
|
+
for origin in origins
|
|
998
|
+
]
|
|
999
|
+
return views[0] if len(views) == 1 else rrb.Horizontal(*views)
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def _scalar_views(log: ReplayLog) -> dict[str, list[tuple[str, int | None]]]:
|
|
1003
|
+
"""Each group's plots, as a view origin and the row it declared, in order.
|
|
1004
|
+
|
|
1005
|
+
Channels whose names differ only in the last segment share one plot, so
|
|
1006
|
+
`joints/0` to `joints/6` are seven series of a plot named `joints` and a
|
|
1007
|
+
channel named without a `/` is a plot of its own. A benchmark chooses between
|
|
1008
|
+
the two by naming its channels, and chooses the rows they sit on by numbering
|
|
1009
|
+
them, which is the whole of what ADR 0008 lets it decide.
|
|
1010
|
+
|
|
1011
|
+
A plot takes the row of the first channel that reaches it, since the channels
|
|
1012
|
+
sharing one are one series and cannot be drawn in two places.
|
|
1013
|
+
"""
|
|
1014
|
+
views: dict[str, list[tuple[str, int | None]]] = {}
|
|
1015
|
+
for channel in log.channels:
|
|
1016
|
+
if channel.kind is not ChannelKind.SCALAR:
|
|
1017
|
+
continue
|
|
1018
|
+
entries = views.setdefault(_scalar_group(channel), [])
|
|
1019
|
+
origin = _scalar_view(channel)
|
|
1020
|
+
if origin not in [seen for seen, _ in entries]:
|
|
1021
|
+
entries.append((origin, channel.row))
|
|
1022
|
+
return views
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _scalar_group(channel: Channel) -> str:
|
|
1026
|
+
"""The tab a scalar channel is plotted in.
|
|
1027
|
+
|
|
1028
|
+
A channel that declares no group falls back to the first segment of its name,
|
|
1029
|
+
so a log that says nothing about layout still gets its scalars sorted.
|
|
1030
|
+
"""
|
|
1031
|
+
return channel.group or channel.name.split("/")[0]
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def _scalar_view(channel: Channel) -> str:
|
|
1035
|
+
"""The origin of the plot a scalar channel's series is drawn in and logged on.
|
|
1036
|
+
|
|
1037
|
+
Under the group, except where the group came from the name's own first
|
|
1038
|
+
segment - prefixing that again would name the tab twice over.
|
|
1039
|
+
"""
|
|
1040
|
+
if channel.group is None:
|
|
1041
|
+
path = f"/{channel.name}"
|
|
1042
|
+
else:
|
|
1043
|
+
path = f"/{channel.group}/{channel.name}"
|
|
1044
|
+
return path.rsplit("/", 1)[0] if "/" in channel.name else path
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
def _series_name(path: str, origins: Iterable[str]) -> str:
|
|
1048
|
+
"""A channel's series name: its path below the outermost plot that draws it.
|
|
1049
|
+
|
|
1050
|
+
A plot draws the plots nested under it as well, so `sensor/table` stays
|
|
1051
|
+
distinct from another sensor's `table` there.
|
|
1052
|
+
"""
|
|
1053
|
+
outer = min(
|
|
1054
|
+
(
|
|
1055
|
+
origin
|
|
1056
|
+
for origin in origins
|
|
1057
|
+
if path == origin or path.startswith(f"{origin}/")
|
|
1058
|
+
),
|
|
1059
|
+
key=len,
|
|
1060
|
+
)
|
|
1061
|
+
return path[len(outer) + 1 :] or path.rsplit("/", 1)[-1]
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _eye_for(log: ReplayLog, survey: _Survey) -> rrb.EyeControls3D:
|
|
1065
|
+
"""An opening camera derived from the extent of what moves.
|
|
1066
|
+
|
|
1067
|
+
Orbital about the centre of that box, backed off along +x by a little over its
|
|
1068
|
+
diagonal and lifted half as far along +z. A hand-placed position and look
|
|
1069
|
+
target would be exactly the per-benchmark constant a generic reader refuses.
|
|
1070
|
+
"""
|
|
1071
|
+
centre, diagonal = _scene_bounds(log, survey)
|
|
1072
|
+
offset = max(diagonal * _EYE_DIAGONALS, 0.5)
|
|
1073
|
+
return rrb.EyeControls3D(
|
|
1074
|
+
kind=rrb.Eye3DKind.Orbital,
|
|
1075
|
+
position=(centre[0] + offset, centre[1], centre[2] + offset * 0.5),
|
|
1076
|
+
look_target=(centre[0], centre[1], centre[2]),
|
|
1077
|
+
eye_up=(0.0, 0.0, 1.0),
|
|
1078
|
+
)
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _scene_bounds(log: ReplayLog, survey: _Survey) -> tuple[np.ndarray, float]:
|
|
1082
|
+
"""The centre and diagonal of the box around the bodies that move.
|
|
1083
|
+
|
|
1084
|
+
Over the geometry, not the body origins: a body's parts sit at offsets in its
|
|
1085
|
+
frame and carry vertices of their own, so a robot whose links share a base
|
|
1086
|
+
origin has an extent its poses alone put at zero. Part rotations are ignored
|
|
1087
|
+
and each part's vertex box is taken axis-aligned, which can only overstate the
|
|
1088
|
+
extent - the eye ends up slightly further back, never inside the scene.
|
|
1089
|
+
|
|
1090
|
+
The scenery is left out. A robosuite scene carries the room it stands in as a
|
|
1091
|
+
body, and a floor nine metres across gives a box the tabletop occupies a
|
|
1092
|
+
fortieth of, so an eye framing the whole scene opens far enough away that the
|
|
1093
|
+
arm and the objects are a few pixels. Every body counts when none of them
|
|
1094
|
+
moved, which is all a single-step episode has to offer.
|
|
1095
|
+
"""
|
|
1096
|
+
if len(survey.first_poses) == 0:
|
|
1097
|
+
return np.zeros(3), 1.0
|
|
1098
|
+
moving = survey.moving
|
|
1099
|
+
framed = moving if len(moving) > 0 else set(survey.first_poses)
|
|
1100
|
+
parts = {body.name: body.parts for body in log.bodies}
|
|
1101
|
+
corners: list[np.ndarray] = []
|
|
1102
|
+
for name, pose in survey.first_poses.items():
|
|
1103
|
+
if name not in framed:
|
|
1104
|
+
continue
|
|
1105
|
+
origin = np.asarray(pose.position, dtype=np.float64)
|
|
1106
|
+
corners.append(origin)
|
|
1107
|
+
for part in parts.get(name, ()):
|
|
1108
|
+
vertices = np.asarray(part.mesh.vertices, dtype=np.float64)
|
|
1109
|
+
if vertices.size == 0:
|
|
1110
|
+
continue
|
|
1111
|
+
offset = origin + np.asarray(part.position, dtype=np.float64)
|
|
1112
|
+
corners.append(offset + vertices.min(axis=0))
|
|
1113
|
+
corners.append(offset + vertices.max(axis=0))
|
|
1114
|
+
stacked = np.stack(corners)
|
|
1115
|
+
low, high = stacked.min(axis=0), stacked.max(axis=0)
|
|
1116
|
+
return (low + high) / 2.0, float(np.linalg.norm(high - low))
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _scenery(log: ReplayLog, survey: _Survey) -> list[str]:
|
|
1120
|
+
"""The bodies the 3D view leaves out: the room the scene stands in.
|
|
1121
|
+
|
|
1122
|
+
A robosuite scene carries its room as an ordinary body, and one of its parts
|
|
1123
|
+
is a slab nine metres across at about the height of a raised arm. It is a
|
|
1124
|
+
wall to look through as well as a box to be dwarfed by, so an eye placed above
|
|
1125
|
+
it sees nothing but it, whatever the eye is derived from.
|
|
1126
|
+
|
|
1127
|
+
The test is size against motion: a body that never moves and is several times
|
|
1128
|
+
the extent of everything that does is the room. A tabletop is about the size
|
|
1129
|
+
of the arm working on it and stays.
|
|
1130
|
+
|
|
1131
|
+
The geometry is only dropped from the view. It stays in the recording, so a
|
|
1132
|
+
viewer that wants the room can put it back.
|
|
1133
|
+
"""
|
|
1134
|
+
moving = survey.moving
|
|
1135
|
+
if len(moving) == 0:
|
|
1136
|
+
return []
|
|
1137
|
+
_, span = _scene_bounds(log, survey)
|
|
1138
|
+
extents = _body_extents(log)
|
|
1139
|
+
return [
|
|
1140
|
+
name
|
|
1141
|
+
for name in survey.first_poses
|
|
1142
|
+
if name not in moving and extents.get(name, 0.0) > span * _SCENERY_DIAGONALS
|
|
1143
|
+
]
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _body_extents(log: ReplayLog) -> dict[str, float]:
|
|
1147
|
+
"""Each body's own diagonal, over the geometry its parts carry."""
|
|
1148
|
+
extents: dict[str, float] = {}
|
|
1149
|
+
for body in log.bodies:
|
|
1150
|
+
corners: list[np.ndarray] = []
|
|
1151
|
+
for part in body.parts:
|
|
1152
|
+
vertices = np.asarray(part.mesh.vertices, dtype=np.float64)
|
|
1153
|
+
if vertices.size == 0:
|
|
1154
|
+
continue
|
|
1155
|
+
offset = np.asarray(part.position, dtype=np.float64)
|
|
1156
|
+
corners.append(offset + vertices.min(axis=0))
|
|
1157
|
+
corners.append(offset + vertices.max(axis=0))
|
|
1158
|
+
if len(corners) == 0:
|
|
1159
|
+
continue
|
|
1160
|
+
stacked = np.stack(corners)
|
|
1161
|
+
extents[body.name] = float(
|
|
1162
|
+
np.linalg.norm(stacked.max(axis=0) - stacked.min(axis=0))
|
|
1163
|
+
)
|
|
1164
|
+
return extents
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def _same_pose(pose: Pose, other: Pose) -> bool:
|
|
1168
|
+
"""Whether two poses are the same to within what counts as standing still."""
|
|
1169
|
+
return bool(
|
|
1170
|
+
np.allclose(pose.position, other.position, atol=_STILL_METRES)
|
|
1171
|
+
and np.allclose(pose.orientation, other.orientation, atol=_STILL_METRES)
|
|
1172
|
+
)
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def _vertex_normals(vertices: np.ndarray, faces: np.ndarray) -> np.ndarray:
|
|
1176
|
+
"""A normal per vertex, averaged over the triangles meeting at it.
|
|
1177
|
+
|
|
1178
|
+
A mesh logged without them is drawn unlit, which on a scene of untextured
|
|
1179
|
+
parts is one flat silhouette per body and no shape at all. The format carries
|
|
1180
|
+
no normals (manifold-sdk ADR 0004) because they are derivable, and this is
|
|
1181
|
+
where they are derived.
|
|
1182
|
+
|
|
1183
|
+
Each triangle's normal is its edge cross product, whose length is twice the
|
|
1184
|
+
triangle's area, so accumulating them unnormalised weights every vertex by the
|
|
1185
|
+
area around it. A degenerate triangle contributes a zero vector and a vertex
|
|
1186
|
+
with nothing but those keeps the up it starts as.
|
|
1187
|
+
"""
|
|
1188
|
+
normals = np.zeros_like(vertices, dtype=np.float32)
|
|
1189
|
+
if len(faces) == 0:
|
|
1190
|
+
return normals
|
|
1191
|
+
corners = vertices[faces.astype(np.int64)]
|
|
1192
|
+
facing = np.cross(
|
|
1193
|
+
corners[:, 1] - corners[:, 0], corners[:, 2] - corners[:, 0]
|
|
1194
|
+
).astype(np.float32)
|
|
1195
|
+
for corner in range(3):
|
|
1196
|
+
np.add.at(normals, faces[:, corner].astype(np.int64), facing)
|
|
1197
|
+
lengths = np.linalg.norm(normals, axis=1, keepdims=True)
|
|
1198
|
+
return np.divide(normals, lengths, out=normals, where=lengths > 0)
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
def _body_path(name: str) -> str:
|
|
1202
|
+
"""The entity path of a scene body's pose."""
|
|
1203
|
+
return f"/{_BODIES_ROOT}/{_entity_name(name)}"
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
def _entity_name(name: str) -> str:
|
|
1207
|
+
"""A body name as one entity-path segment."""
|
|
1208
|
+
return name.replace("/", "_").replace(":", "_")
|