simulo 0.26.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simulo/__init__.py +433 -0
- simulo/_client/__init__.py +6 -0
- simulo/_client/_entrypoint.py +313 -0
- simulo/_client/_mounts.py +25 -0
- simulo/_client/_runner.py +186 -0
- simulo/_client/_secure_downloads.py +1181 -0
- simulo/_client/app.py +1308 -0
- simulo/_client/asset.py +331 -0
- simulo/_client/asset_api.py +517 -0
- simulo/_client/asset_package.py +1103 -0
- simulo/_client/asset_pins.py +187 -0
- simulo/_client/builtin_aliases.py +107 -0
- simulo/_client/bundle.py +254 -0
- simulo/_client/cancel_api.py +104 -0
- simulo/_client/cli.py +9063 -0
- simulo/_client/config.py +186 -0
- simulo/_client/credentials.py +210 -0
- simulo/_client/discovery.py +214 -0
- simulo/_client/export_api.py +212 -0
- simulo/_client/export_bundle.py +296 -0
- simulo/_client/facades.py +581 -0
- simulo/_client/http.py +414 -0
- simulo/_client/identity_api.py +117 -0
- simulo/_client/install_samples.py +267 -0
- simulo/_client/jobs_api.py +224 -0
- simulo/_client/learning.py +393 -0
- simulo/_client/login.py +319 -0
- simulo/_client/mode.py +29 -0
- simulo/_client/outputs.py +116 -0
- simulo/_client/packaging.py +445 -0
- simulo/_client/preflight_api.py +186 -0
- simulo/_client/preflight_render.py +200 -0
- simulo/_client/registry.py +98 -0
- simulo/_client/runtime.py +185 -0
- simulo/_client/runtime_display.py +90 -0
- simulo/_client/seed_ref.py +76 -0
- simulo/_client/stub.py +41 -0
- simulo/_client/submit_api.py +1057 -0
- simulo/_client/templates/__init__.py +21 -0
- simulo/_client/templates/inference/app.py.tmpl +316 -0
- simulo/_client/templates/inference/simuloignore.tmpl +30 -0
- simulo/_client/templates/scenario/app.py.tmpl +93 -0
- simulo/_client/templates/scenario/simuloignore.tmpl +27 -0
- simulo/_client/templates/training/app.py.tmpl +235 -0
- simulo/_client/templates/training/simuloignore.tmpl +29 -0
- simulo/_client/view_fragment.py +21 -0
- simulo/_client/view_session_api.py +122 -0
- simulo/_client/volume.py +71 -0
- simulo/callbacks.py +274 -0
- simulo/py.typed +0 -0
- simulo-0.26.0.dist-info/METADATA +130 -0
- simulo-0.26.0.dist-info/RECORD +55 -0
- simulo-0.26.0.dist-info/WHEEL +5 -0
- simulo-0.26.0.dist-info/entry_points.txt +2 -0
- simulo-0.26.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Submit-time asset pin resolution — the ``simulo run`` half of registry-asset pinning.
|
|
2
|
+
|
|
3
|
+
Collects every registry-asset ref an app declared (its App-level mounts, surfaced
|
|
4
|
+
as the manifest's ``resources``), resolves them to exact versions against the
|
|
5
|
+
control plane in one batch, prints the ``Resolved assets:`` block, applies the
|
|
6
|
+
``--frozen`` / ``--strict-assets`` gates and the deprecated / not-validated
|
|
7
|
+
warnings, and returns the ``[{ref, digest}]`` pins to attach to the job-create
|
|
8
|
+
call as the top-level ``assets`` field.
|
|
9
|
+
|
|
10
|
+
**Package-hash invariance.** Pins are a property of THIS submission, never of the
|
|
11
|
+
package: they ride the job-create wire body (a sibling of ``args``), never enter
|
|
12
|
+
``args`` or the manifest, so a package's content-addressed ``package_id`` can never
|
|
13
|
+
change based on which versions its unpinned refs resolved to. Exactly the
|
|
14
|
+
viewstream / seed rule (``app.py`` ambient signals).
|
|
15
|
+
|
|
16
|
+
The heavy logic (resolution, gating, rendering) lives here rather than in
|
|
17
|
+
``app.py`` so the submit seam stays lean and this is unit-testable in isolation.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import sys
|
|
23
|
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, TextIO
|
|
24
|
+
|
|
25
|
+
from simulo._client.runtime_display import display_runtime
|
|
26
|
+
from simulo._client.submit_api import AssetPinError, SubmitApiClient
|
|
27
|
+
from simulo.interfaces.platform.asset_catalog import DEFAULT_ASSET_RUNTIME, ResolvedAssetPin, parse_asset_ref
|
|
28
|
+
|
|
29
|
+
#: Re-exported from ``simulo.interfaces.platform.asset_catalog`` (current code and tests
|
|
30
|
+
#: "Runtime-string seam", 2026-07-17 — hoisted there so the CLIENT's resolve
|
|
31
|
+
#: request and the VALIDATOR's finalize call can never send different runtime
|
|
32
|
+
#: strings). Kept as a module attribute here too — existing call sites
|
|
33
|
+
#: (``app.py``, tests) import it from this module.
|
|
34
|
+
__all__ = ["DEFAULT_ASSET_RUNTIME", "collect_asset_refs", "resolve_and_pin_assets"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def collect_asset_refs(manifest: Mapping[str, Any]) -> List[str]:
|
|
38
|
+
"""Return the de-duplicated registry-asset refs an app declared, in first-seen
|
|
39
|
+
order.
|
|
40
|
+
|
|
41
|
+
The refs are the ``uri`` of every ``resources`` entry the manifest carries —
|
|
42
|
+
i.e. every App-level mount that is a :class:`simulo.Asset` (``Volume`` mounts
|
|
43
|
+
live in ``volumes``, not here; ``Asset.usd(...)`` / ``.urdf(...)`` model specs
|
|
44
|
+
are not mounts at all). Order-preserving de-dup so the printed block and the
|
|
45
|
+
resolve batch are stable and never resolve the same ref twice.
|
|
46
|
+
"""
|
|
47
|
+
seen: Dict[str, None] = {}
|
|
48
|
+
for resource in manifest.get("resources") or []:
|
|
49
|
+
uri = resource.get("uri") if isinstance(resource, dict) else resource
|
|
50
|
+
if isinstance(uri, str) and uri and uri not in seen:
|
|
51
|
+
seen[uri] = None
|
|
52
|
+
return list(seen)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _short_digest(digest: str) -> str:
|
|
56
|
+
"""``sha256:9c41…`` — the algo prefix + the first 4 hex chars + an ellipsis.
|
|
57
|
+
A digest with no recognisable hex tail is
|
|
58
|
+
rendered verbatim (defensive; the wire form is always ``sha256:<hex>``)."""
|
|
59
|
+
prefix, _, hexpart = digest.partition(":")
|
|
60
|
+
if hexpart:
|
|
61
|
+
return f"{prefix}:{hexpart[:4]}…"
|
|
62
|
+
return digest
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _is_unpinned(ref: str) -> bool:
|
|
66
|
+
"""``True`` if *ref* names no explicit ``:v<N>`` (resolved to latest at submit).
|
|
67
|
+
|
|
68
|
+
Parses through the ONE shared grammar so ``--frozen``'s notion of "pinned"
|
|
69
|
+
can never drift from what the catalog considers a version. A ref that does not
|
|
70
|
+
even parse is treated as unpinned here — it will fail the resolve with a clear
|
|
71
|
+
miss regardless, and ``--frozen`` should not be the thing that accepts it.
|
|
72
|
+
"""
|
|
73
|
+
try:
|
|
74
|
+
return parse_asset_ref(ref).version is None
|
|
75
|
+
except ValueError:
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _resolved_publisher(canonical_ref: str) -> Optional[str]:
|
|
80
|
+
"""The owning-catalog publisher of a resolved canonical ref, or ``None`` if it
|
|
81
|
+
cannot be parsed (defensive — a resolved ref is always canonical)."""
|
|
82
|
+
try:
|
|
83
|
+
return parse_asset_ref(canonical_ref).publisher
|
|
84
|
+
except ValueError:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _render_resolved_block(refs: Sequence[str], resolved: Mapping[str, ResolvedAssetPin]) -> str:
|
|
89
|
+
"""The ``Resolved assets:`` block — the authored (short) ref →
|
|
90
|
+
resolved version + short digest, columns aligned, in the order the refs were
|
|
91
|
+
declared. When the ref OMITTED its publisher (the caller's own org), the
|
|
92
|
+
owning org is named as a trailing ``(org: <slug>)`` label so the human always
|
|
93
|
+
sees which catalog a bare ref resolved in — the short-form + org-label
|
|
94
|
+
display, layered on the canonical pin that is actually recorded."""
|
|
95
|
+
lines = ["Resolved assets:"]
|
|
96
|
+
width = max((len(ref) for ref in refs), default=0)
|
|
97
|
+
for ref in refs:
|
|
98
|
+
pin = resolved[ref]
|
|
99
|
+
row = f" {ref.ljust(width)} -> v{pin.version} ({_short_digest(pin.digest)})"
|
|
100
|
+
try:
|
|
101
|
+
authored_omits_publisher = parse_asset_ref(ref).publisher is None
|
|
102
|
+
except ValueError:
|
|
103
|
+
authored_omits_publisher = False
|
|
104
|
+
if authored_omits_publisher:
|
|
105
|
+
org = _resolved_publisher(pin.canonical_ref)
|
|
106
|
+
if org is not None:
|
|
107
|
+
row += f" (org: {org})"
|
|
108
|
+
lines.append(row)
|
|
109
|
+
return "\n".join(lines)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def resolve_and_pin_assets(
|
|
113
|
+
client: SubmitApiClient,
|
|
114
|
+
refs: Sequence[str],
|
|
115
|
+
*,
|
|
116
|
+
runtime: str = DEFAULT_ASSET_RUNTIME,
|
|
117
|
+
frozen: bool = False,
|
|
118
|
+
strict: bool = False,
|
|
119
|
+
stream: Optional[TextIO] = None,
|
|
120
|
+
) -> List[Dict[str, str]]:
|
|
121
|
+
"""Resolve *refs* to exact pins, print the resolution, gate, and return
|
|
122
|
+
``[{ref, digest}]`` for the job-create ``assets`` field.
|
|
123
|
+
|
|
124
|
+
Ordering mirrors the promise "failures happen in seconds": the
|
|
125
|
+
``--frozen`` unpinned check runs BEFORE any network call, then a single batch
|
|
126
|
+
resolve, then the missing / deprecated / not-validated checks — all before the
|
|
127
|
+
package is uploaded (the caller sequences this ahead of upload).
|
|
128
|
+
|
|
129
|
+
* ``frozen`` — any unpinned ref fails immediately (CI / release explicitness).
|
|
130
|
+
* ``strict`` — the deprecated and not-validated-on-runtime WARNINGS become
|
|
131
|
+
errors.
|
|
132
|
+
|
|
133
|
+
Raises :class:`AssetPinError` (a ``SubmitApiError`` the CLI already renders as
|
|
134
|
+
a clean one-liner) on any frozen / missing / strict failure. Returns ``[]``
|
|
135
|
+
when *refs* is empty (no resolve call is made). The pins carry each version's
|
|
136
|
+
CANONICAL ref (publisher explicit, version pinned) — so the control plane
|
|
137
|
+
records the EXACT version this submit resolved, never re-resolving "latest" in
|
|
138
|
+
the submit transaction (a TOCTOU a new publish between resolve and submit would
|
|
139
|
+
otherwise open).
|
|
140
|
+
"""
|
|
141
|
+
stream = stream if stream is not None else sys.stderr
|
|
142
|
+
refs = list(refs)
|
|
143
|
+
if not refs:
|
|
144
|
+
return []
|
|
145
|
+
|
|
146
|
+
if frozen:
|
|
147
|
+
unpinned = [ref for ref in refs if _is_unpinned(ref)]
|
|
148
|
+
if unpinned:
|
|
149
|
+
raise AssetPinError(
|
|
150
|
+
"--frozen requires every asset to be pinned to an exact version, but these are unpinned:\n"
|
|
151
|
+
+ "".join(f" {ref}\n" for ref in unpinned)
|
|
152
|
+
+ "Pin each to a specific version (e.g. `robot/forklift:v3`) — `simulo asset inspect <ref>` "
|
|
153
|
+
"lists the versions — or drop --frozen."
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
response = client.resolve_assets(refs, runtime)
|
|
157
|
+
resolved, missing = response.resolved, response.missing
|
|
158
|
+
|
|
159
|
+
if missing:
|
|
160
|
+
raise AssetPinError(
|
|
161
|
+
"these assets could not be resolved (they don't exist, or you don't have access to them):\n"
|
|
162
|
+
+ "".join(f" {ref}: {miss.message}\n" for ref, miss in missing.items())
|
|
163
|
+
+ "Check the ref and your active org — `simulo asset list` shows what you can use."
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
print(_render_resolved_block(refs, resolved), file=stream)
|
|
167
|
+
|
|
168
|
+
warnings: List[str] = []
|
|
169
|
+
for ref in refs:
|
|
170
|
+
pin = resolved[ref]
|
|
171
|
+
if pin.deprecated:
|
|
172
|
+
warnings.append(f"{pin.canonical_ref} is deprecated; a newer version may be available.")
|
|
173
|
+
if not pin.validated_on_runtime:
|
|
174
|
+
warnings.append(f"{pin.canonical_ref} has never validated on runtime {display_runtime(runtime)!r}.")
|
|
175
|
+
|
|
176
|
+
if warnings:
|
|
177
|
+
if strict:
|
|
178
|
+
raise AssetPinError(
|
|
179
|
+
"--strict-assets: these asset warnings are treated as errors:\n"
|
|
180
|
+
+ "".join(f" {warning}\n" for warning in warnings)
|
|
181
|
+
+ "Revalidate the asset (`simulo asset validate <ref> --runtime "
|
|
182
|
+
+ f"{runtime}`), pin a validated version, or drop --strict-assets."
|
|
183
|
+
)
|
|
184
|
+
for warning in warnings:
|
|
185
|
+
print(f"warning: {warning}", file=stream)
|
|
186
|
+
|
|
187
|
+
return [{"ref": resolved[ref].canonical_ref, "digest": resolved[ref].digest} for ref in refs]
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""``builtin://`` → global-catalog alias table + submit-time deprecation scan.
|
|
2
|
+
|
|
3
|
+
The legacy ``builtin://<name>`` scheme (``simulo.Asset.usd("builtin://cartpole")``)
|
|
4
|
+
is being absorbed into the global catalog under ``simulo/robot/<name>``. While
|
|
5
|
+
the shim keeps ``builtin://`` refs working, ``simulo run`` prints a
|
|
6
|
+
one-line deprecation notice at submit for every ``builtin://<name>`` whose catalog
|
|
7
|
+
equivalent has been seeded — pointing the user at the ``simulo/robot/...`` idiom.
|
|
8
|
+
|
|
9
|
+
Two pieces:
|
|
10
|
+
|
|
11
|
+
* :data:`BUILTIN_ASSET_ALIASES` — the ``<builtin-name> -> <catalog-ref>`` table.
|
|
12
|
+
**PR-13 populated this table for real**, from a live GPU seeding run
|
|
13
|
+
(``scripts/seed-global-catalog.py``, run against a real control plane + a
|
|
14
|
+
real worker on the local 3090): every entry below is a builtin whose USD
|
|
15
|
+
closure localized AND whose real cloud validation (settle + actuation
|
|
16
|
+
probes) actually PASSED. A ``BUILTIN_ROBOT_CONFIGS`` name that isn't here
|
|
17
|
+
(``cartpole``, ``franka``, ``humanoid``, ``unitree_go2``, ``ur10`` as of
|
|
18
|
+
PR-13) genuinely has NOT been seeded — it failed real validation or could
|
|
19
|
+
not be localized into a self-contained package — so the scan below finds
|
|
20
|
+
nothing for it and NO notice is printed; that ``builtin://`` ref still
|
|
21
|
+
resolves through the legacy path. Do not add entries here by hand: a name
|
|
22
|
+
belongs in this table only once ``scripts/seed-global-catalog.py`` has
|
|
23
|
+
actually published + validated its ``simulo/robot/<name>`` catalog entry
|
|
24
|
+
(the script's own ``update_alias_table`` appends here, additively, keyed
|
|
25
|
+
on the SAME name it published under — see that script for the current
|
|
26
|
+
seed report), or the notice would send users to a ref that 404s.
|
|
27
|
+
|
|
28
|
+
* :func:`scan_source_for_builtin_aliases` — a textual scan of the packaged source
|
|
29
|
+
files at submit, returning every ``(builtin_name, catalog_ref)`` pair whose
|
|
30
|
+
``builtin_name`` both appears in a ``builtin://`` literal in the source AND has
|
|
31
|
+
a table entry. Purely lexical (a substring/regex scan, not an import or an AST
|
|
32
|
+
walk) so it stays torch-free and cannot execute user code.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import re
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Dict, List, Tuple
|
|
40
|
+
|
|
41
|
+
#: ``<builtin-name>`` → the canonical ``simulo/robot/<name>`` catalog ref it maps
|
|
42
|
+
#: to. Populated by ``scripts/seed-global-catalog.py`` (see the module
|
|
43
|
+
#: docstring) — additively, one entry per REAL-validated seeded builtin. The
|
|
44
|
+
#: deprecation notice fires ONLY for a name present in this table.
|
|
45
|
+
BUILTIN_ASSET_ALIASES: Dict[str, str] = {
|
|
46
|
+
"anymal": "simulo/robot/anymal-c",
|
|
47
|
+
"anymal_c": "simulo/robot/anymal-c",
|
|
48
|
+
"h1": "simulo/robot/unitree-h1",
|
|
49
|
+
"jetbot": "simulo/robot/jetbot",
|
|
50
|
+
"pick_and_place": "simulo/robot/pick-and-place",
|
|
51
|
+
"unitree_h1": "simulo/robot/unitree-h1",
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
#: Matches a ``builtin://<name>`` literal in source text. ``<name>`` is a
|
|
55
|
+
#: Python-identifier-shaped ``BUILTIN_ROBOT_CONFIGS`` key (lowercase letters,
|
|
56
|
+
#: digits, underscores — e.g. ``pick_and_place``, ``unitree_h1``, ``anymal_c``)
|
|
57
|
+
#: — DELIBERATELY WIDER than the catalog's own hyphen-only
|
|
58
|
+
#: ``ASSET_SLUG_PATTERN``: this regex captures the AUTHORED ``builtin://``
|
|
59
|
+
#: literal (a registry key), never a catalog ref itself (critic MAJOR, fix
|
|
60
|
+
#: loop 1 — the original ``[a-z0-9-]`` alphabet silently truncated
|
|
61
|
+
#: ``builtin://pick_and_place`` at the first underscore, capturing only
|
|
62
|
+
#: ``"pick"``, which is never a table key, so the notice for every seeded
|
|
63
|
+
#: underscore-named builtin silently never fired). The scan only ACTS on
|
|
64
|
+
#: names that are also in :data:`BUILTIN_ASSET_ALIASES`, so a stray match on
|
|
65
|
+
#: some other alphabet is harmless (it will not be in the table).
|
|
66
|
+
_BUILTIN_URI_RE = re.compile(r"builtin://([a-z0-9][a-z0-9_-]*)")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def scan_source_for_builtin_aliases(package_path: Path) -> List[Tuple[str, str]]:
|
|
70
|
+
"""Return ``(builtin_name, catalog_ref)`` for every seeded ``builtin://`` use
|
|
71
|
+
in the packaged ``.py`` sources under *package_path*.
|
|
72
|
+
|
|
73
|
+
Short-circuits to ``[]`` when :data:`BUILTIN_ASSET_ALIASES` is empty (a
|
|
74
|
+
genuinely-empty table happens only in a checkout where the seeding script
|
|
75
|
+
has never run — the shipped table is populated, see the module
|
|
76
|
+
docstring), so submit pays no scan cost in that case. The
|
|
77
|
+
scan is lexical and best-effort: an unreadable file is skipped, never fatal —
|
|
78
|
+
a deprecation notice must never break a submit. Results are de-duplicated and
|
|
79
|
+
ordered by builtin name for a stable, testable notice.
|
|
80
|
+
"""
|
|
81
|
+
if not BUILTIN_ASSET_ALIASES:
|
|
82
|
+
return []
|
|
83
|
+
found: Dict[str, str] = {}
|
|
84
|
+
for py_file in sorted(package_path.rglob("*.py")):
|
|
85
|
+
try:
|
|
86
|
+
text = py_file.read_text(encoding="utf-8", errors="replace")
|
|
87
|
+
except OSError:
|
|
88
|
+
continue # unreadable file: skip, never fail a submit over the scan
|
|
89
|
+
for match in _BUILTIN_URI_RE.finditer(text):
|
|
90
|
+
name = match.group(1)
|
|
91
|
+
catalog_ref = BUILTIN_ASSET_ALIASES.get(name)
|
|
92
|
+
if catalog_ref is not None:
|
|
93
|
+
found[name] = catalog_ref
|
|
94
|
+
return [(name, found[name]) for name in sorted(found)]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def format_builtin_deprecation_notice(matches: List[Tuple[str, str]]) -> str:
|
|
98
|
+
"""Render the one-block deprecation notice for :func:`scan_source_for_builtin_aliases`'s
|
|
99
|
+
matches. Assumes a non-empty *matches* — callers guard on it."""
|
|
100
|
+
lines = [
|
|
101
|
+
"warning: the builtin:// asset scheme is deprecated and will be removed. "
|
|
102
|
+
"Move to the catalog ref (same asset, one grammar):",
|
|
103
|
+
]
|
|
104
|
+
width = max(len(f"builtin://{name}") for name, _ in matches)
|
|
105
|
+
for name, catalog_ref in matches:
|
|
106
|
+
lines.append(f" {f'builtin://{name}'.ljust(width)} -> {catalog_ref}")
|
|
107
|
+
return "\n".join(lines)
|
simulo/_client/bundle.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Build a v1 manifest from a live ``App`` and write the package (submit).
|
|
2
|
+
|
|
3
|
+
Shared by ``simulo run`` (the CLI) and ``JobFunction.spawn`` — the *submit* path:
|
|
4
|
+
discover the app's metadata and write the package (manifest + source bundle) to
|
|
5
|
+
disk. The manifest builder reads only the public ``App`` surface, so it works in
|
|
6
|
+
discovery mode (where submit runs). ``ensure_bundle`` memoises the written package
|
|
7
|
+
per *submission* — ``(app, job_name, args)`` — so re-submitting the same job with
|
|
8
|
+
the same args reuses the package, while a second spawn with a different job or
|
|
9
|
+
different args writes a distinct package (no clobber).
|
|
10
|
+
|
|
11
|
+
Submit NEVER executes a job and NEVER invokes the backend — it only writes the
|
|
12
|
+
package. This module imports no heavy library; it stays on the torch-free side of
|
|
13
|
+
the seam, and the ``simulo-backend`` runner (the sole executor) consumes the
|
|
14
|
+
package it writes.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import TYPE_CHECKING, Any, Optional, Sequence
|
|
21
|
+
|
|
22
|
+
from simulo._client.asset import Asset
|
|
23
|
+
from simulo._client.packaging import canonical_args, package, read_bundle_package_id
|
|
24
|
+
from simulo._client.volume import Volume
|
|
25
|
+
from simulo.interfaces.platform import is_platform_pinned
|
|
26
|
+
from simulo.interfaces.platform.manifest import Manifest
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING: # typing-only; avoids any import cycle with app.py at runtime
|
|
29
|
+
from simulo._client.app import App, JobSpec
|
|
30
|
+
|
|
31
|
+
#: Default location for spawn-built bundles, relative to the project root. Pruned
|
|
32
|
+
#: from packaging by the ``.simulo`` directory exclude, so it never self-includes.
|
|
33
|
+
DEFAULT_OUT_SUBPATH = (".simulo", "packages")
|
|
34
|
+
|
|
35
|
+
# Memoise the built bundle per *submission* (per process). Keyed on
|
|
36
|
+
# ``(id(app), job_name, canonical_json(args))`` — an App lives for the whole run,
|
|
37
|
+
# so identity reuse is not a concern, and keying on the job + args means multiple
|
|
38
|
+
# spawns in one entrypoint (different job or different args) each write their own
|
|
39
|
+
# package instead of the first one being reused for all.
|
|
40
|
+
#
|
|
41
|
+
# Each entry stores ``(bundle_dir, package_id)`` — the package_id the bundle
|
|
42
|
+
# was written for when THIS process cached it — not just the path. The local
|
|
43
|
+
# bundle directory name is a short, non-injective digest prefix (see
|
|
44
|
+
# ``packaging.bundle_dir_name``), so a directory existing at the cached path is
|
|
45
|
+
# NOT proof it still holds this submission's content: another submission whose
|
|
46
|
+
# id happens to share the same short prefix could have called ``package()``
|
|
47
|
+
# again for a DIFFERENT id and overwritten it (``package()`` unconditionally
|
|
48
|
+
# rmtree+rewrites whatever it finds there). ``ensure_bundle`` re-verifies the
|
|
49
|
+
# manifest on every cache hit before trusting it (see below) — turning a
|
|
50
|
+
# prefix collision into a redundant rebuild, never a silently wrong package.
|
|
51
|
+
#: Cache key's 4th element is the sorted, de-duplicated URIs of every
|
|
52
|
+
#: ambiently-CAPTURED asset visible at this ``ensure_bundle`` call (construction-
|
|
53
|
+
#: capture — see ``asset.py``) — included so a LATER spawn() in the same process,
|
|
54
|
+
#: after the entrypoint has constructed MORE assets since an earlier spawn() with
|
|
55
|
+
#: the identical ``(app, job_name, args)``, is never served a stale cached bundle
|
|
56
|
+
#: whose ``resources`` predates those constructions (``package_id`` itself never
|
|
57
|
+
#: changes with the asset set — see the package-hash-invariance rule — so the
|
|
58
|
+
#: `package_id` re-check below cannot by itself catch this staleness).
|
|
59
|
+
_BUNDLE_CACHE: dict[tuple[int, str, str, tuple[str, ...]], tuple[Path, str]] = {}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _mount_ref(mount: object) -> dict[str, Any]:
|
|
63
|
+
"""Serialise a mount (Volume/Asset) to a manifest reference."""
|
|
64
|
+
if isinstance(mount, Volume):
|
|
65
|
+
return {"type": "volume", "name": mount.name, "create_if_missing": mount.create_if_missing}
|
|
66
|
+
if isinstance(mount, Asset):
|
|
67
|
+
return {"type": "asset", "uri": mount.uri, "read_only": mount.is_read_only}
|
|
68
|
+
return {"type": "unknown", "repr": repr(mount)}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _callback_entry(callback: object) -> dict[str, Any]:
|
|
72
|
+
"""Serialise one job callback to its manifest entry.
|
|
73
|
+
|
|
74
|
+
Duck-typed on ``manifest_entry()`` so ``simulo.callbacks`` classes carry
|
|
75
|
+
their full config into the manifest. Foreign ``JobCallbackProtocol``
|
|
76
|
+
implementations (which have no ``manifest_entry``) fall back to a
|
|
77
|
+
config-less ``{"type": <ClassName>, "config": {}}`` record.
|
|
78
|
+
"""
|
|
79
|
+
entry = getattr(callback, "manifest_entry", None)
|
|
80
|
+
if callable(entry):
|
|
81
|
+
result = entry()
|
|
82
|
+
if isinstance(result, dict):
|
|
83
|
+
return result
|
|
84
|
+
return {"type": type(callback).__name__, "config": {}}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _job_entry(spec: "JobSpec") -> dict[str, Any]:
|
|
88
|
+
return {
|
|
89
|
+
"module": spec.module,
|
|
90
|
+
"qualname": spec.qualname,
|
|
91
|
+
"resources": spec.resources,
|
|
92
|
+
"timeout": spec.timeout,
|
|
93
|
+
"retries": spec.retries,
|
|
94
|
+
"resume": spec.resume.value,
|
|
95
|
+
# List of {"type", "config"} dicts (was a list of class-name strings —
|
|
96
|
+
# the backend runner tolerates both shapes; manifest_version stays 1).
|
|
97
|
+
"callbacks": [_callback_entry(callback) for callback in spec.callbacks],
|
|
98
|
+
"mounts": {key: _mount_ref(mount) for key, mount in spec.mounts.items()},
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_manifest(
|
|
103
|
+
app: "App",
|
|
104
|
+
app_file: Path,
|
|
105
|
+
project_root: Path,
|
|
106
|
+
*,
|
|
107
|
+
args: Optional[dict[str, Any]] = None,
|
|
108
|
+
extra_assets: Sequence[Asset] = (),
|
|
109
|
+
) -> Manifest:
|
|
110
|
+
"""Build the v1 manifest from a live app's metadata (digest stamped later).
|
|
111
|
+
|
|
112
|
+
``args`` records the keyword arguments the submitted job will be executed with
|
|
113
|
+
(the values passed to ``spawn``/``submit``). The backend runner merges these
|
|
114
|
+
over any ``--args-json`` it is given, so a submitted package carries the exact
|
|
115
|
+
invocation the user asked for.
|
|
116
|
+
|
|
117
|
+
``extra_assets`` (construction-capture — ``asset.current_captured_assets()``)
|
|
118
|
+
are unioned into ``resources`` alongside the App's own ``mounts=``-declared
|
|
119
|
+
Assets, deduplicated by ``uri`` — an :class:`Asset` constructed but never
|
|
120
|
+
mounted (e.g. only referenced via ``Robot(asset=forklift)``) still needs a
|
|
121
|
+
``resources`` entry so BOTH the submit-time resolve batch (``asset_pins.
|
|
122
|
+
collect_asset_refs`` reads ``resources``) and the local ``run-package`` mount
|
|
123
|
+
setup (``runner._setup_mounts``, also keyed off ``resources``) see it — an
|
|
124
|
+
unmounted handle must never resolve to a silent zero-pin submit.
|
|
125
|
+
|
|
126
|
+
**Submit-time denylist gate:** every package name declared via the
|
|
127
|
+
runtime's ``pip_install()`` is checked against
|
|
128
|
+
``simulo.interfaces.platform.is_platform_pinned`` — a layer naming a
|
|
129
|
+
package the Simulo base runtime already provides (torch, numpy, the CUDA
|
|
130
|
+
wheels, the ``simulo`` distributions) fails the whole submit here with a
|
|
131
|
+
``SystemExit`` listing the offenders, before any package is written or
|
|
132
|
+
uploaded. Fast-fail by design: a shadowing layer version once broke the
|
|
133
|
+
runtime's compiled camera-sensor bindings in ways undiagnosable from the
|
|
134
|
+
job's own traceback, so this raises — it never skips-and-warns.
|
|
135
|
+
"""
|
|
136
|
+
rel = app_file.resolve().relative_to(project_root)
|
|
137
|
+
module = ".".join(rel.with_suffix("").parts)
|
|
138
|
+
entrypoints = app.entrypoint_names
|
|
139
|
+
entrypoint_name = entrypoints[0] if entrypoints else None
|
|
140
|
+
|
|
141
|
+
volumes: dict[str, Any] = {}
|
|
142
|
+
resource_by_uri: dict[str, dict[str, Any]] = {}
|
|
143
|
+
for mount in app.mounts.values():
|
|
144
|
+
if isinstance(mount, Volume):
|
|
145
|
+
volumes[mount.name] = {"create_if_missing": mount.create_if_missing}
|
|
146
|
+
elif isinstance(mount, Asset):
|
|
147
|
+
resource_by_uri[mount.uri] = {"uri": mount.uri, "read_only": mount.is_read_only}
|
|
148
|
+
for asset in extra_assets:
|
|
149
|
+
resource_by_uri.setdefault(asset.uri, {"uri": asset.uri, "read_only": asset.is_read_only})
|
|
150
|
+
resources: list[Any] = list(resource_by_uri.values())
|
|
151
|
+
|
|
152
|
+
runtime = app.runtime
|
|
153
|
+
pip_dependencies = [dict(layer) for layer in getattr(runtime, "pip_dependencies", [])]
|
|
154
|
+
pinned = list(
|
|
155
|
+
dict.fromkeys( # de-duplicate across layers, preserving declaration order
|
|
156
|
+
spec for layer in pip_dependencies for spec in layer.get("packages", []) if is_platform_pinned(str(spec))
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
if pinned:
|
|
160
|
+
raise SystemExit(
|
|
161
|
+
"Cannot pip_install these — the Simulo base runtime already provides them: "
|
|
162
|
+
f"{', '.join(pinned)}. Remove them; the base image includes them."
|
|
163
|
+
)
|
|
164
|
+
return Manifest(
|
|
165
|
+
source_digest="", # stamped by package()
|
|
166
|
+
runtime_id=str(getattr(runtime, "runtime_id", "")),
|
|
167
|
+
# The sub-key DELIBERATELY keeps its pre-0.15 wire spelling even though
|
|
168
|
+
# the decorator it records is now ``@app.entrypoint``. Renaming it
|
|
169
|
+
# changes the manifest bytes — and therefore ``archive_sha256`` — while
|
|
170
|
+
# ``package_id`` (sha256 over source_digest + job + args only) holds
|
|
171
|
+
# still. package_id-same + archive-different is EXACTLY the server's
|
|
172
|
+
# ``package_conflict`` 409 (jobs/service.py), so a rename here
|
|
173
|
+
# hard-blocks every 0.14.x user resubmitting an UNCHANGED
|
|
174
|
+
# entrypoint-less file (an entrypoint-bearing file dodges it only
|
|
175
|
+
# because editing the decorator moves source_digest). The key is an
|
|
176
|
+
# opaque serialization detail nothing reads (the backend runner reads
|
|
177
|
+
# only ``entrypoint.module``); manifest-shape migration is Plan A's
|
|
178
|
+
# (#540 — register-conflict handling). Pinned by
|
|
179
|
+
# test_manifest_wire_stability.py.
|
|
180
|
+
entrypoint={"file": rel.as_posix(), "module": module, "local_entrypoint": entrypoint_name},
|
|
181
|
+
runtime_env=dict(getattr(runtime, "runtime_env", {})),
|
|
182
|
+
jobs={name: _job_entry(spec) for name, spec in app.jobs.items()},
|
|
183
|
+
resources=resources,
|
|
184
|
+
volumes=volumes,
|
|
185
|
+
pip_dependencies=pip_dependencies,
|
|
186
|
+
args=dict(args or {}),
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def ensure_bundle(
|
|
191
|
+
app: "App",
|
|
192
|
+
app_file: Path,
|
|
193
|
+
project_root: Path,
|
|
194
|
+
*,
|
|
195
|
+
job_name: str = "",
|
|
196
|
+
args: Optional[dict[str, Any]] = None,
|
|
197
|
+
out_dir: Optional[Path] = None,
|
|
198
|
+
extra_assets: Sequence[Asset] = (),
|
|
199
|
+
) -> Path:
|
|
200
|
+
"""Write the package for one submission (memoised); return its path.
|
|
201
|
+
|
|
202
|
+
This is the *submit* primitive: the manifest is built from the live app object
|
|
203
|
+
(no second discovery import) with the submitted job's ``args`` recorded, and
|
|
204
|
+
the source tree is written to ``<out>/<bundle_dir_name(package_id)>/`` — the
|
|
205
|
+
per-submission directory keyed on source + ``job_name`` + ``args`` (named by
|
|
206
|
+
the id's short, filesystem-safe form; see ``packaging.bundle_dir_name``).
|
|
207
|
+
Memoised per submission (``(id(app), job_name, canonical_json(args), sorted
|
|
208
|
+
extra_assets uris)``) so re-submitting the same job with the same args AND the
|
|
209
|
+
same captured-asset set reuses the package, while a different job, different
|
|
210
|
+
args, or a DIFFERENT captured-asset set (more constructions since an earlier
|
|
211
|
+
spawn() in the same process — see ``asset.current_captured_assets()``) writes a
|
|
212
|
+
distinct package rather than serving a bundle whose ``resources`` predates a
|
|
213
|
+
since-constructed handle.
|
|
214
|
+
|
|
215
|
+
A cache HIT is re-verified, not trusted blindly: the short bundle directory
|
|
216
|
+
name is not injective (Windows MAX_PATH trade-off — see
|
|
217
|
+
``packaging.bundle_dir_name``), so before returning a cached path this reads
|
|
218
|
+
the manifest actually on disk there and confirms its ``package_id`` still
|
|
219
|
+
matches what THIS cache entry was written for. A mismatch — some other
|
|
220
|
+
submission's id collided on the same short prefix and ``package()``
|
|
221
|
+
overwrote the directory — is treated as a MISS: the package is rebuilt
|
|
222
|
+
(deterministically restoring the correct content for this exact
|
|
223
|
+
submission), never served as a silently wrong package.
|
|
224
|
+
"""
|
|
225
|
+
extra_asset_uris = tuple(sorted({asset.uri for asset in extra_assets}))
|
|
226
|
+
cache_key = (id(app), job_name, canonical_args(args), extra_asset_uris)
|
|
227
|
+
cached = _BUNDLE_CACHE.get(cache_key)
|
|
228
|
+
if cached is not None:
|
|
229
|
+
cached_dir, cached_package_id = cached
|
|
230
|
+
if cached_dir.is_dir() and read_bundle_package_id(cached_dir) == cached_package_id:
|
|
231
|
+
return cached_dir
|
|
232
|
+
if out_dir is None:
|
|
233
|
+
out_dir = project_root.joinpath(*DEFAULT_OUT_SUBPATH)
|
|
234
|
+
manifest = build_manifest(app, app_file, project_root, args=args, extra_assets=extra_assets)
|
|
235
|
+
bundle = package(app_file, out_dir=out_dir, project_root=project_root, manifest=manifest, job_name=job_name)
|
|
236
|
+
bundle_package_id = read_bundle_package_id(bundle)
|
|
237
|
+
if bundle_package_id is None:
|
|
238
|
+
# Internal invariant, not a user-facing input error: package() always
|
|
239
|
+
# stamps package_id into the manifest it writes (see packaging.py). A
|
|
240
|
+
# bare `assert` here would be stripped under `python -O`, silently
|
|
241
|
+
# caching a bundle keyed on a package_id we never actually confirmed
|
|
242
|
+
# (critic finding, PR #363 round-2 re-review, 2026-07-12) — use a
|
|
243
|
+
# real, always-live exception instead.
|
|
244
|
+
raise RuntimeError(
|
|
245
|
+
f"Internal error: package() wrote a bundle at {bundle} with no package_id in its manifest. "
|
|
246
|
+
"This should never happen — please report it."
|
|
247
|
+
)
|
|
248
|
+
_BUNDLE_CACHE[cache_key] = (bundle, bundle_package_id)
|
|
249
|
+
return bundle
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def clear_bundle_cache() -> None:
|
|
253
|
+
"""Drop the per-process bundle cache. Intended for tests."""
|
|
254
|
+
_BUNDLE_CACHE.clear()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""HTTP client for the job cancel endpoint (stdlib only, torch-free).
|
|
2
|
+
|
|
3
|
+
Wire contract (job-cancel-and-demo-apps plan; the control
|
|
4
|
+
plane's half of this route is built by a parallel effort, so this client codes
|
|
5
|
+
directly against the plan's PINNED contract rather than waiting on it):
|
|
6
|
+
``POST /v1/jobs/{job_id}/cancel`` (Cognito bearer, org-scoped).
|
|
7
|
+
|
|
8
|
+
* 200 ``{"job_id": ..., "status": "cancelled"}`` — a QUEUED job is cancelled
|
|
9
|
+
immediately (it can never be claimed by a worker after this).
|
|
10
|
+
* 202 ``{"job_id": ..., "status": "running", "cancel_requested": true}`` — a
|
|
11
|
+
RUNNING job has cancellation requested; the worker confirms termination
|
|
12
|
+
within one heartbeat interval. Idempotent — a repeat cancel of an
|
|
13
|
+
already-cancel-requested running job returns the same 202 shape again.
|
|
14
|
+
* 409 ``job_not_running`` — the job is already in a terminal state
|
|
15
|
+
(completed/failed/cancelled); the response message carries the actual
|
|
16
|
+
status, but the caller (the thin client's CLI module) does not surface it
|
|
17
|
+
verbatim — it reuses the same friendly "has ended" copy ``simulo view``
|
|
18
|
+
uses for the identical situation.
|
|
19
|
+
* 404 ``job_not_found`` — unknown or cross-org job id (anti-enumeration;
|
|
20
|
+
never 403). This is the cancel route's ONLY pinned 404 code — mirrors
|
|
21
|
+
``view_session_api.py``'s contract, and the caller relies on that the same
|
|
22
|
+
way: any OTHER 404 code means the ``/cancel`` route itself isn't deployed
|
|
23
|
+
in this environment (e.g. prod before this feature ships there), not that
|
|
24
|
+
the job is missing — see ``cli._CANCEL_UNAVAILABLE_MESSAGE``.
|
|
25
|
+
|
|
26
|
+
Error responses use the standard ``{"error": {"code", "message", "request_id"}}``
|
|
27
|
+
envelope, parsed by ``http.py`` exactly like every other client in this
|
|
28
|
+
package.
|
|
29
|
+
|
|
30
|
+
The route path is a LOCAL constant here rather than imported from
|
|
31
|
+
``simulo.interfaces.platform.runs`` — deliberately, the same reconciliation
|
|
32
|
+
call ``view_session_api.py`` made: the control-plane cancel route is being
|
|
33
|
+
built by a parallel PR and may land independently of this one, so this client
|
|
34
|
+
codes directly against the plan's pinned path/response shape rather than
|
|
35
|
+
sharing a file with that PR (avoiding a merge conflict). Reconcile into a
|
|
36
|
+
shared ``simulo.interfaces.platform`` constant in a follow-up once that PR has
|
|
37
|
+
merged, if a shared module materializes for it.
|
|
38
|
+
|
|
39
|
+
Shares request plumbing (structured-error parsing, the https-when-token
|
|
40
|
+
guard, the foreign-host bearer rule) with ``jobs_api.py``/``view_session_api.py``
|
|
41
|
+
via ``http.py`` — one request path, not reimplemented a fourth time.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
from typing import Any, Optional
|
|
47
|
+
|
|
48
|
+
from simulo._client import http
|
|
49
|
+
|
|
50
|
+
#: ``POST`` — request cancellation of a queued or running job. ``.format(job_id=...)``.
|
|
51
|
+
CANCEL_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/cancel"
|
|
52
|
+
|
|
53
|
+
_REQUEST_TIMEOUT_S = 10.0 # every outbound call has an explicit timeout (NFR)
|
|
54
|
+
_UNAVAILABLE_HINT = "Check SIMULO_API_URL / SIMULO_ENV, and that you are logged in (`simulo login`)."
|
|
55
|
+
|
|
56
|
+
#: Required fields of a well-formed cancel response (see module docstring).
|
|
57
|
+
_REQUIRED_FIELDS = ("job_id", "status")
|
|
58
|
+
|
|
59
|
+
# Backward-compatible-style aliases — same convention as jobs_api.py /
|
|
60
|
+
# view_session_api.py: these are literally ``http.HttpError``/``http.HttpHTTPError``
|
|
61
|
+
# (not new subclasses), so callers that already catch ``JobsApiError`` (also
|
|
62
|
+
# ``http.HttpError``) catch these too without any extra wiring.
|
|
63
|
+
CancelApiError = http.HttpError
|
|
64
|
+
CancelApiHTTPError = http.HttpHTTPError
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class CancelApiClient:
|
|
68
|
+
"""Client for ``POST /v1/jobs/{job_id}/cancel``."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, base_url: str, *, token: Optional[str] = None) -> None:
|
|
71
|
+
if not base_url.startswith(("http://", "https://")):
|
|
72
|
+
raise CancelApiError(f"API base URL must be an http(s) URL, got {base_url!r}.")
|
|
73
|
+
self._base_url = base_url.rstrip("/")
|
|
74
|
+
self._token = token
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def base_url(self) -> str:
|
|
78
|
+
return self._base_url
|
|
79
|
+
|
|
80
|
+
def cancel_job(self, job_id: str) -> dict[str, Any]:
|
|
81
|
+
"""Request cancellation of *job_id*; returns the parsed 200/202 body.
|
|
82
|
+
|
|
83
|
+
Raises :class:`CancelApiHTTPError` on any non-2xx response — the
|
|
84
|
+
caller (the thin client's CLI module) classifies by ``exc.code``, never by
|
|
85
|
+
``exc.status`` alone (both ``job_not_running`` and the route-missing
|
|
86
|
+
degradation share statuses with other codes, exactly like
|
|
87
|
+
``ViewSessionApiClient.create_view_session``'s callers already do).
|
|
88
|
+
"""
|
|
89
|
+
path = CANCEL_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
|
|
90
|
+
payload = http.request_json(
|
|
91
|
+
"POST",
|
|
92
|
+
self._base_url + path,
|
|
93
|
+
token=self._token,
|
|
94
|
+
api_base_url=self._base_url,
|
|
95
|
+
json_body={},
|
|
96
|
+
timeout=_REQUEST_TIMEOUT_S,
|
|
97
|
+
unavailable_hint=_UNAVAILABLE_HINT,
|
|
98
|
+
)
|
|
99
|
+
if not isinstance(payload, dict):
|
|
100
|
+
raise CancelApiError("Malformed cancel response: expected a JSON object.")
|
|
101
|
+
missing = [field for field in _REQUIRED_FIELDS if field not in payload]
|
|
102
|
+
if missing:
|
|
103
|
+
raise CancelApiError(f"Malformed cancel response: missing field(s) {missing}.")
|
|
104
|
+
return payload
|