revfly 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. revfly/__init__.py +23 -0
  2. revfly/__main__.py +6 -0
  3. revfly-2.0.0.dist-info/METADATA +106 -0
  4. revfly-2.0.0.dist-info/RECORD +37 -0
  5. revfly-2.0.0.dist-info/WHEEL +5 -0
  6. revfly-2.0.0.dist-info/entry_points.txt +3 -0
  7. revfly-2.0.0.dist-info/top_level.txt +2 -0
  8. sensory_translator_v2/__init__.py +116 -0
  9. sensory_translator_v2/assets.py +924 -0
  10. sensory_translator_v2/backends/__init__.py +19 -0
  11. sensory_translator_v2/backends/base.py +484 -0
  12. sensory_translator_v2/backends/mujoco_worker.py +1213 -0
  13. sensory_translator_v2/backends/persistent_mujoco.py +592 -0
  14. sensory_translator_v2/backends/v1_subprocess.py +364 -0
  15. sensory_translator_v2/backends/worker_protocol.py +108 -0
  16. sensory_translator_v2/bio/__init__.py +39 -0
  17. sensory_translator_v2/bio/lif_interface.py +624 -0
  18. sensory_translator_v2/bio/malecns_pools.py +204 -0
  19. sensory_translator_v2/bio/spiking.py +310 -0
  20. sensory_translator_v2/contracts.py +708 -0
  21. sensory_translator_v2/engine/__init__.py +28 -0
  22. sensory_translator_v2/engine/__main__.py +6 -0
  23. sensory_translator_v2/engine/benchmark.py +346 -0
  24. sensory_translator_v2/engine/cli.py +677 -0
  25. sensory_translator_v2/engine/core.py +1269 -0
  26. sensory_translator_v2/engine/multijoint.py +1003 -0
  27. sensory_translator_v2/engine/openapi.py +385 -0
  28. sensory_translator_v2/engine/server.py +1431 -0
  29. sensory_translator_v2/evidence.py +84 -0
  30. sensory_translator_v2/goals.py +172 -0
  31. sensory_translator_v2/rate/__init__.py +60 -0
  32. sensory_translator_v2/rate/bridge.py +424 -0
  33. sensory_translator_v2/rate/model.py +350 -0
  34. sensory_translator_v2/rate/multijoint.py +805 -0
  35. sensory_translator_v2/rate/optim.py +57 -0
  36. sensory_translator_v2/rate/trainer.py +618 -0
  37. sensory_translator_v2/runtime_install.py +370 -0
@@ -0,0 +1,924 @@
1
+ """Content-addressed asset store, v1 compatibility adapter, and XML profile transforms."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import shutil
9
+ import sys
10
+ import tempfile
11
+ import xml.etree.ElementTree as ET
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from .contracts import (
17
+ AssetValidationError,
18
+ EXPECTED_TEACHER_SHA256,
19
+ EXPECTED_XML_SHA256,
20
+ PROFILES,
21
+ require_profile,
22
+ )
23
+
24
+ ENV_ASSET_ROOT = "SENSORY_TRANSLATOR_V2_ASSET_ROOT"
25
+ ENV_RUNTIME_ROOT = "SENSORY_TRANSLATOR_V2_RUNTIME_ROOT"
26
+ ENV_PHYSICS_PYTHON = "SENSORY_TRANSLATOR_V2_PHYSICS_PYTHON"
27
+ ENV_SCRIPTS_ROOT = "SENSORY_TRANSLATOR_V2_SCRIPTS_ROOT"
28
+
29
+ BUNDLE_SCHEMA = "sensory_translator_v2_asset_bundle_v1"
30
+
31
+ V1_XML_RELPATH = Path("external/FlyMimic/flymimic/assets/models/best_combined_arm_cvt3.xml")
32
+ V1_MESH_DIR_RELPATH = Path("external/FlyMimic/flymimic/assets/models/meshes/stl")
33
+ V1_TEACHER_RELPATH = Path("results/flymimic_teacher/demo_20261027_v1/teacher_rollout.npz")
34
+
35
+ KNOWN_V1_MODELS: tuple[dict[str, Any], ...] = (
36
+ {
37
+ "model_id": "20261111_v2_preclip",
38
+ "rel_dir": "results/conditional_rate_ffn/20261111_v2_preclip",
39
+ "profile": "original_v1",
40
+ "expected_checkpoint_sha256": "7b88ba8e9f9946c28869fb571c27074058d419d34ffbeb1e9fc70cfea5c736cf",
41
+ "default_for_profile": "original_v1",
42
+ "description": "Preclip conditional rate FFN trained on 7 offline trajectories",
43
+ },
44
+ {
45
+ "model_id": "20261113_v5_trust",
46
+ "rel_dir": "results/rate_ffn_e2e/20261113_v5_trust",
47
+ "profile": "adjacent_static_parent_v1",
48
+ "expected_checkpoint_sha256": "52f96d066bedc0445f9f41a35e29895df33fb796ffe46fd3d3a67b8056ee42c0",
49
+ "default_for_profile": None,
50
+ "description": "End-to-end physical VJP rate FFN (v5 trust-region stage)",
51
+ },
52
+ {
53
+ "model_id": "20261114_v6_longrun",
54
+ "rel_dir": "results/rate_ffn_e2e/20261114_v6_longrun",
55
+ "profile": "adjacent_static_parent_v1",
56
+ "expected_checkpoint_sha256": "5a84401a4e966a3eb81df14197cbe065200f3c66ba1933da929e3a1539320f2c",
57
+ "default_for_profile": "adjacent_static_parent_v1",
58
+ "description": "Resumed end-to-end physical VJP rate FFN (112 accepted updates)",
59
+ },
60
+ )
61
+
62
+ KNOWN_V2_MODELS: tuple[dict[str, Any], ...] = (
63
+ {
64
+ "model_id": "20261115_v2_fastbridge_lowenergy",
65
+ "rel_dir": "results/v2_models/20261115_v2_fastbridge_lowenergy",
66
+ "profile": "adjacent_static_parent_v1",
67
+ "expected_checkpoint_sha256": "4a60a9ba41a4c19f27efd5486a27e94387f0d6b622e5ac9b2f44a1dd9e939a8f",
68
+ "default_for_profile": None,
69
+ "description": "Stage 3 V2 fast-bridge (tau=2.0ms, K=105) low-energy E2E VJP rate FFN",
70
+ },
71
+ )
72
+
73
+ V2_MODELS_RELPATH = Path("results/v2_models")
74
+ MULTIJOINT_MODEL_ID = "20261116_v2_multijoint15_resnorm"
75
+ MULTIJOINT_CHECKPOINT_RELPATH = V2_MODELS_RELPATH / MULTIJOINT_MODEL_ID / "best_model.npz"
76
+ MULTIJOINT_CHECKPOINT_SHA256 = "45879505e182e205b3554903a25a7e89362304ada2a307f65e1a01731f5a3585"
77
+ # Checkout resolution tries the current checkpoint first and supports older pinned bundles.
78
+ KNOWN_MULTIJOINT_CHECKPOINTS = {
79
+ MULTIJOINT_MODEL_ID: MULTIJOINT_CHECKPOINT_SHA256,
80
+ "20261115_v2_multijoint15_fastbridge": "8548c7c7aa8227c4a695518327d576b2c6cf26afe7e549e0c5bc1510d01424b9",
81
+ }
82
+
83
+ ADJACENT_EXCLUDED_BODY_PAIRS: tuple[tuple[str, str], ...] = (
84
+ ("Thorax", "LFCoxa"),
85
+ ("Thorax", "RFCoxa"),
86
+ )
87
+
88
+
89
+ def sha256_bytes(payload: bytes) -> str:
90
+ """Return lowercase hexadecimal SHA256 digest of bytes."""
91
+ return hashlib.sha256(payload).hexdigest()
92
+
93
+
94
+ def sha256_file(path: str | Path) -> str:
95
+ """Return lowercase hexadecimal SHA256 digest of a file."""
96
+ file_path = Path(path)
97
+ if not file_path.is_file():
98
+ raise AssetValidationError(f"Asset file does not exist: {file_path}")
99
+ digest = hashlib.sha256()
100
+ with file_path.open("rb") as handle:
101
+ for chunk in iter(lambda: handle.read(65_536), b""):
102
+ digest.update(chunk)
103
+ return digest.hexdigest()
104
+
105
+
106
+ def canonical_json_sha256(payload: Any) -> str:
107
+ """Hash canonical UTF-8 JSON without machine-dependent formatting."""
108
+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
109
+ return sha256_bytes(encoded)
110
+
111
+
112
+ def profile_transform_spec(profile: str) -> dict[str, Any]:
113
+ """Return the versioned deterministic rule describing a physics profile transform."""
114
+ require_profile(profile)
115
+ if profile == "original_v1":
116
+ spec = {
117
+ "schema": "sensory_translator_v2_profile_rule_v1",
118
+ "profile": "original_v1",
119
+ "rule_version": 1,
120
+ "enable_multiccd": False,
121
+ "excluded_body_pairs": [],
122
+ "xml_rewrite": "identity",
123
+ }
124
+ elif profile == "multiccd_v1":
125
+ spec = {
126
+ "schema": "sensory_translator_v2_profile_rule_v1",
127
+ "profile": "multiccd_v1",
128
+ "rule_version": 1,
129
+ "enable_multiccd": True,
130
+ "excluded_body_pairs": [],
131
+ "xml_rewrite": "identity_with_runtime_multiccd_flag",
132
+ }
133
+ else:
134
+ spec = {
135
+ "schema": "sensory_translator_v2_profile_rule_v1",
136
+ "profile": "adjacent_static_parent_v1",
137
+ "rule_version": 1,
138
+ "enable_multiccd": True,
139
+ "excluded_body_pairs": [list(pair) for pair in ADJACENT_EXCLUDED_BODY_PAIRS],
140
+ "xml_rewrite": "inject_contact_exclude_pairs",
141
+ }
142
+ spec["profile_rule_sha256"] = canonical_json_sha256(spec)
143
+ return spec
144
+
145
+
146
+ def transform_xml_for_profile(
147
+ raw_xml_text: str,
148
+ profile: str,
149
+ *,
150
+ mesh_dir_override: str | None = None,
151
+ ) -> str:
152
+ """Deterministically materialize XML text for a profile without altering raw assets."""
153
+ require_profile(profile)
154
+ needs_rewrite = profile == "adjacent_static_parent_v1" or mesh_dir_override is not None
155
+ if not needs_rewrite:
156
+ return raw_xml_text
157
+
158
+ root = ET.fromstring(raw_xml_text)
159
+ if mesh_dir_override is not None:
160
+ base_dir = Path(mesh_dir_override)
161
+ for mesh in root.findall(".//asset/mesh"):
162
+ file_attr = mesh.attrib.get("file")
163
+ if file_attr:
164
+ mesh_name = Path(file_attr).name
165
+ mesh.attrib["file"] = str(base_dir / mesh_name)
166
+
167
+ if profile == "adjacent_static_parent_v1":
168
+ contact = root.find("contact")
169
+ if contact is None:
170
+ contact = ET.SubElement(root, "contact")
171
+ existing = {
172
+ (el.attrib.get("body1"), el.attrib.get("body2"))
173
+ for el in contact.findall("exclude")
174
+ }
175
+ for body1, body2 in ADJACENT_EXCLUDED_BODY_PAIRS:
176
+ if (body1, body2) not in existing and (body2, body1) not in existing:
177
+ ET.SubElement(contact, "exclude", {"body1": body1, "body2": body2})
178
+
179
+ return ET.tostring(root, encoding="unicode")
180
+
181
+
182
+ @dataclass(frozen=True)
183
+ class ModelEntry:
184
+ """Resolved rate policy checkpoint entry in an asset catalog."""
185
+
186
+ model_id: str
187
+ profile: str
188
+ checkpoint_path: Path
189
+ checkpoint_sha256: str
190
+ preregistered_config_path: Path | None = None
191
+ preregistered_config_sha256: str | None = None
192
+ selection_path: Path | None = None
193
+ selection_sha256: str | None = None
194
+ default_for_profile: str | None = None
195
+ description: str = ""
196
+
197
+ def to_dict(self) -> dict[str, Any]:
198
+ return {
199
+ "model_id": self.model_id,
200
+ "profile": self.profile,
201
+ "checkpoint_path": str(self.checkpoint_path),
202
+ "checkpoint_sha256": self.checkpoint_sha256,
203
+ "preregistered_config_sha256": self.preregistered_config_sha256,
204
+ "selection_sha256": self.selection_sha256,
205
+ "default_for_profile": self.default_for_profile,
206
+ "description": self.description,
207
+ }
208
+
209
+
210
+ @dataclass(frozen=True)
211
+ class AssetCatalog:
212
+ """Immutable verified view of all assets from either a v2 bundle or v1 checkout."""
213
+
214
+ source_kind: str # "v2_bundle" | "v1_checkout"
215
+ asset_root: Path
216
+ runtime_root: Path
217
+ physics_python: Path
218
+ scripts_root: Path | None
219
+ xml_path: Path
220
+ raw_xml_sha256: str
221
+ mesh_paths: dict[str, Path] # relative key (e.g. "meshes/stl/Thorax.stl") -> path
222
+ mesh_hashes: dict[str, str]
223
+ mesh_tree_sha256: str
224
+ teacher_path: Path
225
+ teacher_sha256: str
226
+ models: dict[str, ModelEntry]
227
+ manifest_sha256: str
228
+ provenance: dict[str, Any] = field(default_factory=dict)
229
+ _bundled_multijoint_model: ModelEntry | None = None
230
+
231
+ def default_model_for_profile(self, profile: str) -> ModelEntry:
232
+ require_profile(profile)
233
+ for model in self.models.values():
234
+ if model.default_for_profile == profile:
235
+ return model
236
+ for model in self.models.values():
237
+ if model.profile == profile:
238
+ return model
239
+ # If no model was trained specifically for multiccd_v1, fall back deterministically.
240
+ if "20261114_v6_longrun" in self.models:
241
+ return self.models["20261114_v6_longrun"]
242
+ if self.models:
243
+ return next(iter(self.models.values()))
244
+ raise AssetValidationError("Asset catalog contains no registered rate models")
245
+
246
+ @property
247
+ def v2_models(self) -> dict[str, ModelEntry]:
248
+ """Discover optional V2-trained models in checkout workspaces with strict SHA-256 checks."""
249
+ if self.source_kind != "v1_checkout":
250
+ return {}
251
+ found: dict[str, ModelEntry] = {}
252
+ for spec in KNOWN_V2_MODELS:
253
+ ckpt_path = self.asset_root / spec["rel_dir"] / "best_model.npz"
254
+ if not ckpt_path.is_file():
255
+ continue
256
+ ckpt_sha = sha256_file(ckpt_path)
257
+ expected_ckpt = spec["expected_checkpoint_sha256"]
258
+ if expected_ckpt and ckpt_sha != expected_ckpt:
259
+ raise AssetValidationError(
260
+ f"v2 checkpoint sha256 mismatch for {spec['model_id']}: expected {expected_ckpt}, got {ckpt_sha}"
261
+ )
262
+ found[spec["model_id"]] = ModelEntry(
263
+ model_id=spec["model_id"],
264
+ profile=spec["profile"],
265
+ checkpoint_path=ckpt_path,
266
+ checkpoint_sha256=ckpt_sha,
267
+ preregistered_config_path=None,
268
+ preregistered_config_sha256=None,
269
+ selection_path=None,
270
+ selection_sha256=None,
271
+ default_for_profile=spec["default_for_profile"],
272
+ description=spec["description"],
273
+ )
274
+ return found
275
+
276
+ @property
277
+ def all_models(self) -> dict[str, ModelEntry]:
278
+ """Combined catalog of frozen V1 baseline models and registered V2 models."""
279
+ return {**self.models, **self.v2_models}
280
+
281
+ def resolve_multijoint_model(self) -> ModelEntry | None:
282
+ """Return the selected multi-joint model and verify its registered hash."""
283
+ model = self._bundled_multijoint_model
284
+ if self.source_kind == "v1_checkout":
285
+ for model_id, digest in KNOWN_MULTIJOINT_CHECKPOINTS.items():
286
+ checkpoint = self.asset_root / V2_MODELS_RELPATH / model_id / "best_model.npz"
287
+ if checkpoint.is_file():
288
+ model = ModelEntry(model_id, "adjacent_static_parent_v1", checkpoint, digest)
289
+ break
290
+ if model is None:
291
+ return None
292
+ actual = sha256_file(model.checkpoint_path)
293
+ if actual != model.checkpoint_sha256:
294
+ raise AssetValidationError(
295
+ f"Multi-joint checkpoint sha256 mismatch for {model.model_id}: "
296
+ f"expected {model.checkpoint_sha256}, got {actual}"
297
+ )
298
+ return model
299
+
300
+ def resolve_multijoint_checkpoint(self) -> Path | None:
301
+ """Return the selected multi-joint checkpoint path when available."""
302
+ model = self.resolve_multijoint_model()
303
+ return model.checkpoint_path if model is not None else None
304
+
305
+ def resolve_model(self, model_id: str | None, *, profile: str) -> ModelEntry:
306
+ require_profile(profile)
307
+ if model_id is None:
308
+ return self.default_model_for_profile(profile)
309
+ available = self.all_models
310
+ if model_id not in available:
311
+ raise AssetValidationError(
312
+ f"Unknown model_id {model_id!r}; available: {sorted(available.keys())}"
313
+ )
314
+ return available[model_id]
315
+
316
+ def three_hash_record(self, profile: str, *, materialized_xml_sha256: str | None = None) -> dict[str, Any]:
317
+ """Explicitly separate upstream raw hashes, profile rule hash, and materialized XML hash."""
318
+ spec = profile_transform_spec(profile)
319
+ return {
320
+ "raw_xml_sha256": self.raw_xml_sha256,
321
+ "mesh_tree_sha256": self.mesh_tree_sha256,
322
+ "teacher_sha256": self.teacher_sha256,
323
+ "profile_name": profile,
324
+ "profile_rule_sha256": spec["profile_rule_sha256"],
325
+ "materialized_xml_sha256": materialized_xml_sha256,
326
+ "asset_manifest_sha256": self.manifest_sha256,
327
+ }
328
+
329
+
330
+ def _find_repo_checkout(start: Path | None = None) -> Path | None:
331
+ candidates = []
332
+ if start is not None:
333
+ candidates.append(Path(start).resolve())
334
+ candidates.append(Path.cwd().resolve())
335
+ here = Path(__file__).resolve()
336
+ candidates.extend(here.parents)
337
+ seen: set[Path] = set()
338
+ for candidate in candidates:
339
+ for parent in (candidate, *candidate.parents):
340
+ if parent in seen:
341
+ continue
342
+ seen.add(parent)
343
+ if (parent / V1_XML_RELPATH).is_file() and (parent / V1_TEACHER_RELPATH).is_file():
344
+ return parent
345
+ return None
346
+
347
+
348
+ def default_runtime_asset_root() -> Path:
349
+ """Directory managed by the runtime bundle installer."""
350
+ configured = Path(os.environ.get("XDG_DATA_HOME", "")).expanduser()
351
+ data_home = configured if configured.is_absolute() else Path.home() / ".local/share"
352
+ return data_home / "sensory-translator-v2/bundle"
353
+
354
+
355
+ def resolve_asset_root(asset_root: str | Path | None = None) -> Path:
356
+ """Resolve an explicit root, environment override, checkout, or installed bundle."""
357
+ if asset_root is not None:
358
+ return Path(asset_root).expanduser().resolve()
359
+ env_val = os.environ.get(ENV_ASSET_ROOT)
360
+ if env_val:
361
+ return Path(env_val).expanduser().resolve()
362
+ checkout = _find_repo_checkout()
363
+ if checkout is not None:
364
+ return checkout
365
+ installed = default_runtime_asset_root()
366
+ if installed.is_dir():
367
+ return installed.resolve()
368
+ raise AssetValidationError(
369
+ f"No runtime bundle found. Run `revfly install-assets <release.zip>` "
370
+ f"or pass --asset-root (or set {ENV_ASSET_ROOT})."
371
+ )
372
+
373
+
374
+ def resolve_runtime_root(runtime_root: str | Path | None = None, *, asset_root: Path | None = None) -> Path:
375
+ """Resolve runtime output/cache root outside source trees."""
376
+ if runtime_root is not None:
377
+ return Path(runtime_root).expanduser().resolve()
378
+ env_val = os.environ.get(ENV_RUNTIME_ROOT)
379
+ if env_val:
380
+ return Path(env_val).expanduser().resolve()
381
+ if asset_root is not None and (asset_root / "results").is_dir():
382
+ return (asset_root / "results/v2_runtime").resolve()
383
+ return (Path.home() / ".cache/sensory_translator_v2").resolve()
384
+
385
+
386
+ def _preserve_symlink_abspath(path: str | Path) -> Path:
387
+ """Resolve user/relative segments without dereferencing Python venv symlinks."""
388
+ return Path(os.path.abspath(os.path.expanduser(str(path))))
389
+
390
+
391
+ def resolve_physics_python(
392
+ physics_python: str | Path | None = None,
393
+ *,
394
+ asset_root: Path | None = None,
395
+ ) -> Path:
396
+ """Resolve Python executable holding MuJoCo + dm_control without breaking venv symlinks."""
397
+ if physics_python is not None:
398
+ return _preserve_symlink_abspath(physics_python)
399
+ env_val = os.environ.get(ENV_PHYSICS_PYTHON)
400
+ if env_val:
401
+ return _preserve_symlink_abspath(env_val)
402
+ search_bases: list[Path] = []
403
+ if asset_root is not None:
404
+ search_bases.append(asset_root)
405
+ checkout = _find_repo_checkout()
406
+ if checkout is not None:
407
+ search_bases.append(checkout)
408
+ for base in search_bases:
409
+ venv_py = base / ".venv-flymimic/bin/python"
410
+ if venv_py.is_file():
411
+ return _preserve_symlink_abspath(venv_py)
412
+ return _preserve_symlink_abspath(sys.executable)
413
+
414
+
415
+ def resolve_scripts_root(
416
+ scripts_root: str | Path | None = None,
417
+ *,
418
+ asset_root: Path | None = None,
419
+ ) -> Path | None:
420
+ """Resolve v1 frozen scripts directory when available (for v1_subprocess backend only)."""
421
+ if scripts_root is not None:
422
+ cand = Path(scripts_root).expanduser().resolve()
423
+ return cand if cand.is_dir() else None
424
+ env_val = os.environ.get(ENV_SCRIPTS_ROOT)
425
+ if env_val:
426
+ cand = Path(env_val).expanduser().resolve()
427
+ return cand if cand.is_dir() else None
428
+ for base in ([asset_root] if asset_root is not None else []) + [Path.cwd().resolve()]:
429
+ cand = base / "scripts"
430
+ if (cand / "differentiate_neuromuscle_plant.py").is_file():
431
+ return cand.resolve()
432
+ checkout = _find_repo_checkout()
433
+ if checkout is not None and (checkout / "scripts/differentiate_neuromuscle_plant.py").is_file():
434
+ return (checkout / "scripts").resolve()
435
+ return None
436
+
437
+
438
+ def _compute_mesh_tree_sha256(mesh_hashes: dict[str, str]) -> str:
439
+ return canonical_json_sha256(sorted(mesh_hashes.items()))
440
+
441
+
442
+ def _load_v2_bundle(
443
+ root: Path,
444
+ *,
445
+ runtime_root: Path,
446
+ physics_python: Path,
447
+ scripts_root: Path | None,
448
+ verify_hashes: bool,
449
+ ) -> AssetCatalog:
450
+ manifest_path = root / "manifest.json"
451
+ if not manifest_path.is_file():
452
+ raise AssetValidationError(f"v2 asset bundle missing manifest.json: {manifest_path}")
453
+ try:
454
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
455
+ except Exception as exc:
456
+ raise AssetValidationError(f"Invalid asset bundle JSON {manifest_path}: {exc}") from None
457
+ if manifest.get("schema_version") != BUNDLE_SCHEMA:
458
+ raise AssetValidationError(
459
+ f"Unsupported bundle schema {manifest.get('schema_version')!r}; expected {BUNDLE_SCHEMA!r}"
460
+ )
461
+
462
+ def _obj_path(sha: str, label: str) -> Path:
463
+ if not isinstance(sha, str) or len(sha) != 64:
464
+ raise AssetValidationError(f"Malformed sha256 for {label}: {sha!r}")
465
+ obj = root / "objects" / sha[:2] / sha
466
+ if not obj.is_file():
467
+ raise AssetValidationError(f"Missing content-addressed asset {label} at {obj}")
468
+ if verify_hashes:
469
+ actual = sha256_file(obj)
470
+ if actual != sha:
471
+ raise AssetValidationError(
472
+ f"Hash mismatch for {label} ({obj}): expected {sha}, got {actual}"
473
+ )
474
+ return obj
475
+
476
+ fly_xml = manifest.get("raw_xml", {})
477
+ raw_xml_sha = str(fly_xml.get("sha256", ""))
478
+ if raw_xml_sha != EXPECTED_XML_SHA256:
479
+ raise AssetValidationError(
480
+ f"Bundle XML hash {raw_xml_sha} != expected frozen FlyMimic XML {EXPECTED_XML_SHA256}"
481
+ )
482
+ xml_path = _obj_path(raw_xml_sha, "raw_xml")
483
+
484
+ stl_map = manifest.get("stl_meshes", {})
485
+ if len(stl_map) != 71:
486
+ raise AssetValidationError(f"Expected 71 STL meshes in bundle, found {len(stl_map)}")
487
+ mesh_paths: dict[str, Path] = {}
488
+ mesh_hashes: dict[str, str] = {}
489
+ for rel_key, digest in sorted(stl_map.items()):
490
+ mesh_paths[rel_key] = _obj_path(str(digest), rel_key)
491
+ mesh_hashes[rel_key] = str(digest)
492
+ mesh_tree_sha = _compute_mesh_tree_sha256(mesh_hashes)
493
+ if manifest.get("mesh_tree_sha256") != mesh_tree_sha:
494
+ raise AssetValidationError("Bundle mesh_tree_sha256 mismatch")
495
+
496
+ teacher_info = manifest.get("teacher", {})
497
+ teacher_sha = str(teacher_info.get("sha256", ""))
498
+ if teacher_sha != EXPECTED_TEACHER_SHA256:
499
+ raise AssetValidationError(
500
+ f"Bundle teacher hash {teacher_sha} != expected {EXPECTED_TEACHER_SHA256}"
501
+ )
502
+ teacher_path = _obj_path(teacher_sha, "teacher")
503
+
504
+ v2_checkpoint_hashes = {spec["model_id"]: spec["expected_checkpoint_sha256"] for spec in KNOWN_V2_MODELS}
505
+ models: dict[str, ModelEntry] = {}
506
+ for model_id, entry in sorted(manifest.get("models", {}).items()):
507
+ ckpt_sha = str(entry["checkpoint_sha256"])
508
+ expected_ckpt = v2_checkpoint_hashes.get(model_id)
509
+ if verify_hashes and expected_ckpt and ckpt_sha != expected_ckpt:
510
+ raise AssetValidationError(
511
+ f"v2 checkpoint sha256 mismatch for {model_id}: expected {expected_ckpt}, got {ckpt_sha}"
512
+ )
513
+ ckpt_path = _obj_path(ckpt_sha, f"model:{model_id}")
514
+ prereg_sha = entry.get("preregistered_config_sha256")
515
+ sel_sha = entry.get("selection_sha256")
516
+ models[model_id] = ModelEntry(
517
+ model_id=model_id,
518
+ profile=require_profile(entry["profile"]),
519
+ checkpoint_path=ckpt_path,
520
+ checkpoint_sha256=ckpt_sha,
521
+ preregistered_config_path=_obj_path(prereg_sha, f"{model_id}:prereg") if prereg_sha else None,
522
+ preregistered_config_sha256=prereg_sha,
523
+ selection_path=_obj_path(sel_sha, f"{model_id}:selection") if sel_sha else None,
524
+ selection_sha256=sel_sha,
525
+ default_for_profile=entry.get("default_for_profile"),
526
+ description=str(entry.get("description", "")),
527
+ )
528
+
529
+ multijoint_model = None
530
+ multijoint_info = manifest.get("multijoint_checkpoint")
531
+ if multijoint_info is not None:
532
+ if not isinstance(multijoint_info, dict) or not isinstance(multijoint_info.get("model_id"), str):
533
+ raise AssetValidationError("Invalid registered multi-joint checkpoint metadata in bundle")
534
+ model_id = multijoint_info["model_id"]
535
+ digest = KNOWN_MULTIJOINT_CHECKPOINTS.get(model_id)
536
+ if digest is None or multijoint_info.get("sha256") != digest:
537
+ raise AssetValidationError("Invalid registered multi-joint checkpoint metadata in bundle")
538
+ checkpoint = _obj_path(digest, "multi-joint checkpoint")
539
+ multijoint_model = ModelEntry(model_id, "adjacent_static_parent_v1", checkpoint, digest)
540
+
541
+ logical = {
542
+ "schema_version": "sensory_translator_v2_asset_catalog_v1",
543
+ "raw_xml": {"sha256": raw_xml_sha},
544
+ "mesh_tree_sha256": mesh_tree_sha,
545
+ "teacher": {"sha256": teacher_sha},
546
+ "models": {
547
+ mid: {
548
+ "profile": m.profile,
549
+ "checkpoint_sha256": m.checkpoint_sha256,
550
+ "preregistered_config_sha256": m.preregistered_config_sha256,
551
+ "selection_sha256": m.selection_sha256,
552
+ "default_for_profile": m.default_for_profile,
553
+ }
554
+ for mid, m in sorted(models.items())
555
+ },
556
+ }
557
+ if multijoint_model is not None:
558
+ logical["multijoint_checkpoint"] = {
559
+ "model_id": multijoint_model.model_id,
560
+ "sha256": multijoint_model.checkpoint_sha256,
561
+ }
562
+ return AssetCatalog(
563
+ source_kind="v2_bundle",
564
+ asset_root=root,
565
+ runtime_root=runtime_root,
566
+ physics_python=physics_python,
567
+ scripts_root=scripts_root,
568
+ xml_path=xml_path,
569
+ raw_xml_sha256=raw_xml_sha,
570
+ mesh_paths=mesh_paths,
571
+ mesh_hashes=mesh_hashes,
572
+ mesh_tree_sha256=mesh_tree_sha,
573
+ teacher_path=teacher_path,
574
+ teacher_sha256=teacher_sha,
575
+ models=models,
576
+ manifest_sha256=canonical_json_sha256(logical),
577
+ provenance={"bundle_manifest": str(manifest_path)},
578
+ _bundled_multijoint_model=multijoint_model,
579
+ )
580
+
581
+
582
+ def _load_v1_checkout(
583
+ root: Path,
584
+ *,
585
+ runtime_root: Path,
586
+ physics_python: Path,
587
+ scripts_root: Path | None,
588
+ verify_hashes: bool,
589
+ ) -> AssetCatalog:
590
+ xml_path = root / V1_XML_RELPATH
591
+ mesh_dir = root / V1_MESH_DIR_RELPATH
592
+ teacher_path = root / V1_TEACHER_RELPATH
593
+ if not xml_path.is_file():
594
+ raise AssetValidationError(f"FlyMimic XML not found under {root}: {xml_path}")
595
+ if not mesh_dir.is_dir():
596
+ raise AssetValidationError(f"FlyMimic STL mesh dir not found under {root}: {mesh_dir}")
597
+ if not teacher_path.is_file():
598
+ raise AssetValidationError(f"Frozen teacher rollout not found under {root}: {teacher_path}")
599
+
600
+ raw_xml_sha = sha256_file(xml_path)
601
+ if raw_xml_sha != EXPECTED_XML_SHA256:
602
+ raise AssetValidationError(
603
+ f"Raw XML sha256 mismatch: expected {EXPECTED_XML_SHA256}, got {raw_xml_sha}"
604
+ )
605
+ teacher_sha = sha256_file(teacher_path)
606
+ if teacher_sha != EXPECTED_TEACHER_SHA256:
607
+ raise AssetValidationError(
608
+ f"Teacher sha256 mismatch: expected {EXPECTED_TEACHER_SHA256}, got {teacher_sha}"
609
+ )
610
+
611
+ stl_files = sorted(mesh_dir.glob("*.stl"))
612
+ if len(stl_files) != 71:
613
+ raise AssetValidationError(f"Expected 71 STL meshes in {mesh_dir}, found {len(stl_files)}")
614
+ mesh_paths: dict[str, Path] = {}
615
+ mesh_hashes: dict[str, str] = {}
616
+ for stl in stl_files:
617
+ rel_key = f"meshes/stl/{stl.name}"
618
+ mesh_paths[rel_key] = stl
619
+ mesh_hashes[rel_key] = sha256_file(stl)
620
+ mesh_tree_sha = _compute_mesh_tree_sha256(mesh_hashes)
621
+
622
+ models: dict[str, ModelEntry] = {}
623
+ for spec in KNOWN_V1_MODELS:
624
+ model_dir = root / spec["rel_dir"]
625
+ ckpt_path = model_dir / "best_model.npz"
626
+ if not ckpt_path.is_file():
627
+ continue
628
+ ckpt_sha = sha256_file(ckpt_path)
629
+ expected_ckpt = spec["expected_checkpoint_sha256"]
630
+ if verify_hashes and expected_ckpt and ckpt_sha != expected_ckpt:
631
+ raise AssetValidationError(
632
+ f"v1 checkpoint sha256 mismatch for {spec['model_id']}: expected {expected_ckpt}, got {ckpt_sha}"
633
+ )
634
+ prereg = model_dir / "preregistered_config.json"
635
+ sel = model_dir / "selection_before_physics.json"
636
+ models[spec["model_id"]] = ModelEntry(
637
+ model_id=spec["model_id"],
638
+ profile=spec["profile"],
639
+ checkpoint_path=ckpt_path,
640
+ checkpoint_sha256=ckpt_sha,
641
+ preregistered_config_path=prereg if prereg.is_file() else None,
642
+ preregistered_config_sha256=sha256_file(prereg) if prereg.is_file() else None,
643
+ selection_path=sel if sel.is_file() else None,
644
+ selection_sha256=sha256_file(sel) if sel.is_file() else None,
645
+ default_for_profile=spec["default_for_profile"],
646
+ description=spec["description"],
647
+ )
648
+ if not models:
649
+ raise AssetValidationError(f"No valid rate checkpoints discovered under {root}")
650
+
651
+ logical = {
652
+ "schema_version": "sensory_translator_v2_asset_catalog_v1",
653
+ "raw_xml": {"sha256": raw_xml_sha},
654
+ "mesh_tree_sha256": mesh_tree_sha,
655
+ "teacher": {"sha256": teacher_sha},
656
+ "models": {
657
+ mid: {
658
+ "profile": m.profile,
659
+ "checkpoint_sha256": m.checkpoint_sha256,
660
+ "preregistered_config_sha256": m.preregistered_config_sha256,
661
+ "selection_sha256": m.selection_sha256,
662
+ "default_for_profile": m.default_for_profile,
663
+ }
664
+ for mid, m in sorted(models.items())
665
+ },
666
+ }
667
+ return AssetCatalog(
668
+ source_kind="v1_checkout",
669
+ asset_root=root,
670
+ runtime_root=runtime_root,
671
+ physics_python=physics_python,
672
+ scripts_root=scripts_root,
673
+ xml_path=xml_path,
674
+ raw_xml_sha256=raw_xml_sha,
675
+ mesh_paths=mesh_paths,
676
+ mesh_hashes=mesh_hashes,
677
+ mesh_tree_sha256=mesh_tree_sha,
678
+ teacher_path=teacher_path,
679
+ teacher_sha256=teacher_sha,
680
+ models=models,
681
+ manifest_sha256=canonical_json_sha256(logical),
682
+ provenance={"v1_root": str(root)},
683
+ )
684
+
685
+
686
+ def load_asset_catalog(
687
+ *,
688
+ asset_root: str | Path | None = None,
689
+ runtime_root: str | Path | None = None,
690
+ physics_python: str | Path | None = None,
691
+ scripts_root: str | Path | None = None,
692
+ verify_hashes: bool = True,
693
+ ) -> AssetCatalog:
694
+ """Load and verify assets from either a content-addressed v2 bundle or a v1 checkout."""
695
+ resolved_asset_root = resolve_asset_root(asset_root)
696
+ resolved_runtime_root = resolve_runtime_root(runtime_root, asset_root=resolved_asset_root)
697
+ resolved_physics_py = resolve_physics_python(physics_python, asset_root=resolved_asset_root)
698
+
699
+ if (resolved_asset_root / "manifest.json").is_file() and (resolved_asset_root / "objects").is_dir():
700
+ explicit_scripts = (
701
+ Path(scripts_root).expanduser().resolve()
702
+ if scripts_root is not None
703
+ else (
704
+ Path(os.environ[ENV_SCRIPTS_ROOT]).expanduser().resolve()
705
+ if os.environ.get(ENV_SCRIPTS_ROOT)
706
+ else None
707
+ )
708
+ )
709
+ return _load_v2_bundle(
710
+ resolved_asset_root,
711
+ runtime_root=resolved_runtime_root,
712
+ physics_python=resolved_physics_py,
713
+ scripts_root=explicit_scripts if (explicit_scripts and explicit_scripts.is_dir()) else None,
714
+ verify_hashes=verify_hashes,
715
+ )
716
+ resolved_scripts = resolve_scripts_root(scripts_root, asset_root=resolved_asset_root)
717
+ return _load_v1_checkout(
718
+ resolved_asset_root,
719
+ runtime_root=resolved_runtime_root,
720
+ physics_python=resolved_physics_py,
721
+ scripts_root=resolved_scripts,
722
+ verify_hashes=verify_hashes,
723
+ )
724
+
725
+
726
+ def verify_asset_catalog(catalog: AssetCatalog) -> dict[str, Any]:
727
+ """Re-verify every asset hash on disk in the loaded catalog."""
728
+ raw_xml = sha256_file(catalog.xml_path)
729
+ if raw_xml != catalog.raw_xml_sha256 or raw_xml != EXPECTED_XML_SHA256:
730
+ raise AssetValidationError(f"XML hash mismatch during verify: {raw_xml}")
731
+ teacher_sha = sha256_file(catalog.teacher_path)
732
+ if teacher_sha != catalog.teacher_sha256 or teacher_sha != EXPECTED_TEACHER_SHA256:
733
+ raise AssetValidationError(f"Teacher hash mismatch during verify: {teacher_sha}")
734
+ checked = 2
735
+ for rel_key, mesh_path in catalog.mesh_paths.items():
736
+ actual = sha256_file(mesh_path)
737
+ expected = catalog.mesh_hashes[rel_key]
738
+ if actual != expected:
739
+ raise AssetValidationError(f"STL hash mismatch for {rel_key}: {actual} != {expected}")
740
+ checked += 1
741
+ for mid, entry in catalog.models.items():
742
+ actual = sha256_file(entry.checkpoint_path)
743
+ if actual != entry.checkpoint_sha256:
744
+ raise AssetValidationError(f"Checkpoint hash mismatch for {mid}")
745
+ checked += 1
746
+ report: dict[str, Any] = {
747
+ "verified": True,
748
+ "source_kind": catalog.source_kind,
749
+ "asset_root": str(catalog.asset_root),
750
+ "checked_files": checked,
751
+ "raw_xml_sha256": raw_xml,
752
+ "mesh_tree_sha256": catalog.mesh_tree_sha256,
753
+ "teacher_sha256": teacher_sha,
754
+ "manifest_sha256": catalog.manifest_sha256,
755
+ "models": {mid: entry.to_dict() for mid, entry in sorted(catalog.models.items())},
756
+ }
757
+ multijoint_model = catalog.resolve_multijoint_model() if catalog.source_kind == "v2_bundle" else None
758
+ if multijoint_model is not None:
759
+ report["checked_files"] += 1
760
+ report["multijoint_checkpoint"] = {
761
+ "model_id": multijoint_model.model_id,
762
+ "sha256": multijoint_model.checkpoint_sha256,
763
+ "checkpoint_path": str(multijoint_model.checkpoint_path),
764
+ }
765
+ return report
766
+
767
+
768
+ def export_asset_bundle(
769
+ destination: str | Path,
770
+ *,
771
+ asset_root: str | Path | None = None,
772
+ overwrite: bool = False,
773
+ include_v2_models: bool = False,
774
+ ) -> dict[str, Any]:
775
+ """Export verified assets, with optional inclusion of registered V2 checkpoints."""
776
+ catalog = load_asset_catalog(asset_root=asset_root, verify_hashes=True)
777
+ models = catalog.all_models if include_v2_models else catalog.models
778
+ multijoint_model = (
779
+ catalog.resolve_multijoint_model()
780
+ if include_v2_models or catalog.source_kind == "v2_bundle" else None
781
+ )
782
+ if include_v2_models and multijoint_model is None and not any(
783
+ spec["model_id"] in models for spec in KNOWN_V2_MODELS
784
+ ):
785
+ raise AssetValidationError(f"No registered V2 checkpoints found under {catalog.asset_root}")
786
+ dest = Path(destination).expanduser().resolve()
787
+ if dest.exists():
788
+ if not overwrite and any(dest.iterdir()):
789
+ raise AssetValidationError(f"Bundle destination is non-empty: {dest}")
790
+ dest.mkdir(parents=True, exist_ok=True)
791
+ objects_dir = dest / "objects"
792
+ objects_dir.mkdir(parents=True, exist_ok=True)
793
+
794
+ def _store(src: Path, digest: str) -> None:
795
+ sub = objects_dir / digest[:2]
796
+ sub.mkdir(parents=True, exist_ok=True)
797
+ target = sub / digest
798
+ if target.is_file() and sha256_file(target) == digest:
799
+ return
800
+ fd, tmp_name = tempfile.mkstemp(dir=sub, prefix=f".{digest[:12]}.", suffix=".tmp")
801
+ os.close(fd)
802
+ tmp_path = Path(tmp_name)
803
+ try:
804
+ shutil.copy2(src, tmp_path)
805
+ tmp_path.replace(target)
806
+ except Exception:
807
+ if tmp_path.exists():
808
+ tmp_path.unlink()
809
+ raise
810
+
811
+ _store(catalog.xml_path, catalog.raw_xml_sha256)
812
+ _store(catalog.teacher_path, catalog.teacher_sha256)
813
+ for rel_key, mesh_path in catalog.mesh_paths.items():
814
+ _store(mesh_path, catalog.mesh_hashes[rel_key])
815
+
816
+ model_manifest: dict[str, Any] = {}
817
+ for mid, model in sorted(models.items()):
818
+ _store(model.checkpoint_path, model.checkpoint_sha256)
819
+ if model.preregistered_config_path and model.preregistered_config_sha256:
820
+ _store(model.preregistered_config_path, model.preregistered_config_sha256)
821
+ if model.selection_path and model.selection_sha256:
822
+ _store(model.selection_path, model.selection_sha256)
823
+ model_manifest[mid] = {
824
+ "profile": model.profile,
825
+ "checkpoint_sha256": model.checkpoint_sha256,
826
+ "preregistered_config_sha256": model.preregistered_config_sha256,
827
+ "selection_sha256": model.selection_sha256,
828
+ "default_for_profile": model.default_for_profile,
829
+ "description": model.description,
830
+ }
831
+
832
+ manifest_payload = {
833
+ "schema_version": BUNDLE_SCHEMA,
834
+ "raw_xml": {
835
+ "relative_path": "best_combined_arm_cvt3.xml",
836
+ "sha256": catalog.raw_xml_sha256,
837
+ },
838
+ "stl_meshes": dict(sorted(catalog.mesh_hashes.items())),
839
+ "mesh_tree_sha256": catalog.mesh_tree_sha256,
840
+ "teacher": {
841
+ "relative_path": "teacher_rollout.npz",
842
+ "sha256": catalog.teacher_sha256,
843
+ },
844
+ "models": model_manifest,
845
+ }
846
+ if multijoint_model is not None:
847
+ _store(multijoint_model.checkpoint_path, multijoint_model.checkpoint_sha256)
848
+ manifest_payload["multijoint_checkpoint"] = {
849
+ "model_id": multijoint_model.model_id,
850
+ "sha256": multijoint_model.checkpoint_sha256,
851
+ }
852
+ manifest_tmp = dest / ".manifest.json.tmp"
853
+ manifest_tmp.write_text(
854
+ json.dumps(manifest_payload, indent=2, sort_keys=True) + "\n",
855
+ encoding="utf-8",
856
+ )
857
+ manifest_tmp.replace(dest / "manifest.json")
858
+ reloaded = load_asset_catalog(asset_root=dest, verify_hashes=True)
859
+ return verify_asset_catalog(reloaded)
860
+
861
+
862
+ def materialize_profile_model(
863
+ catalog: AssetCatalog,
864
+ profile: str,
865
+ *,
866
+ cache_base: Path | None = None,
867
+ ) -> dict[str, Any]:
868
+ """Ensure a filesystem XML + sibling meshes/stl directory exists for MuJoCo loading.
869
+
870
+ Using a deterministic directory keyed by `(raw_xml_sha256, mesh_tree_sha256, profile_rule_sha256)`
871
+ keeps materialized relative mesh references intact without hardcoding checkout paths into XML.
872
+ """
873
+ spec = profile_transform_spec(profile)
874
+ base = (cache_base or (catalog.runtime_root / "materialized")).resolve()
875
+ key = sha256_bytes(
876
+ f"{catalog.raw_xml_sha256}:{catalog.mesh_tree_sha256}:{spec['profile_rule_sha256']}".encode("utf-8")
877
+ )[:24]
878
+ model_dir = base / f"{profile}_{key}"
879
+ mesh_out_dir = model_dir / "meshes/stl"
880
+ xml_out_path = model_dir / "model.xml"
881
+ done_marker = model_dir / ".ready.json"
882
+
883
+ raw_xml_text = catalog.xml_path.read_text(encoding="utf-8")
884
+ materialized_xml_text = transform_xml_for_profile(raw_xml_text, profile)
885
+ materialized_xml_sha = sha256_bytes(materialized_xml_text.encode("utf-8"))
886
+
887
+ if not (done_marker.is_file() and xml_out_path.is_file() and sha256_file(xml_out_path) == materialized_xml_sha):
888
+ mesh_out_dir.mkdir(parents=True, exist_ok=True)
889
+ for rel_key, src_mesh in catalog.mesh_paths.items():
890
+ dst_mesh = model_dir / rel_key
891
+ if not dst_mesh.is_file() or sha256_file(dst_mesh) != catalog.mesh_hashes[rel_key]:
892
+ fd, tmp_name = tempfile.mkstemp(dir=dst_mesh.parent, prefix=".stl.", suffix=".tmp")
893
+ os.close(fd)
894
+ tmp_path = Path(tmp_name)
895
+ try:
896
+ shutil.copy2(src_mesh, tmp_path)
897
+ tmp_path.replace(dst_mesh)
898
+ except Exception:
899
+ if tmp_path.exists():
900
+ tmp_path.unlink()
901
+ raise
902
+ xml_tmp = xml_out_path.with_suffix(".xml.tmp")
903
+ xml_tmp.write_text(materialized_xml_text, encoding="utf-8")
904
+ xml_tmp.replace(xml_out_path)
905
+ marker_tmp = done_marker.with_suffix(".json.tmp")
906
+ marker_tmp.write_text(
907
+ json.dumps(
908
+ catalog.three_hash_record(profile, materialized_xml_sha256=materialized_xml_sha),
909
+ indent=2,
910
+ sort_keys=True,
911
+ ),
912
+ encoding="utf-8",
913
+ )
914
+ marker_tmp.replace(done_marker)
915
+
916
+ return {
917
+ "xml_path": xml_out_path,
918
+ "xml_text": materialized_xml_text,
919
+ "model_dir": model_dir,
920
+ "three_hashes": catalog.three_hash_record(
921
+ profile,
922
+ materialized_xml_sha256=materialized_xml_sha,
923
+ ),
924
+ }