simulo 0.26.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simulo/__init__.py +433 -0
- simulo/_client/__init__.py +6 -0
- simulo/_client/_entrypoint.py +313 -0
- simulo/_client/_mounts.py +25 -0
- simulo/_client/_runner.py +186 -0
- simulo/_client/_secure_downloads.py +1181 -0
- simulo/_client/app.py +1308 -0
- simulo/_client/asset.py +331 -0
- simulo/_client/asset_api.py +517 -0
- simulo/_client/asset_package.py +1103 -0
- simulo/_client/asset_pins.py +187 -0
- simulo/_client/builtin_aliases.py +107 -0
- simulo/_client/bundle.py +254 -0
- simulo/_client/cancel_api.py +104 -0
- simulo/_client/cli.py +9063 -0
- simulo/_client/config.py +186 -0
- simulo/_client/credentials.py +210 -0
- simulo/_client/discovery.py +214 -0
- simulo/_client/export_api.py +212 -0
- simulo/_client/export_bundle.py +296 -0
- simulo/_client/facades.py +581 -0
- simulo/_client/http.py +414 -0
- simulo/_client/identity_api.py +117 -0
- simulo/_client/install_samples.py +267 -0
- simulo/_client/jobs_api.py +224 -0
- simulo/_client/learning.py +393 -0
- simulo/_client/login.py +319 -0
- simulo/_client/mode.py +29 -0
- simulo/_client/outputs.py +116 -0
- simulo/_client/packaging.py +445 -0
- simulo/_client/preflight_api.py +186 -0
- simulo/_client/preflight_render.py +200 -0
- simulo/_client/registry.py +98 -0
- simulo/_client/runtime.py +185 -0
- simulo/_client/runtime_display.py +90 -0
- simulo/_client/seed_ref.py +76 -0
- simulo/_client/stub.py +41 -0
- simulo/_client/submit_api.py +1057 -0
- simulo/_client/templates/__init__.py +21 -0
- simulo/_client/templates/inference/app.py.tmpl +316 -0
- simulo/_client/templates/inference/simuloignore.tmpl +30 -0
- simulo/_client/templates/scenario/app.py.tmpl +93 -0
- simulo/_client/templates/scenario/simuloignore.tmpl +27 -0
- simulo/_client/templates/training/app.py.tmpl +235 -0
- simulo/_client/templates/training/simuloignore.tmpl +29 -0
- simulo/_client/view_fragment.py +21 -0
- simulo/_client/view_session_api.py +122 -0
- simulo/_client/volume.py +71 -0
- simulo/callbacks.py +274 -0
- simulo/py.typed +0 -0
- simulo-0.26.0.dist-info/METADATA +130 -0
- simulo-0.26.0.dist-info/RECORD +55 -0
- simulo-0.26.0.dist-info/WHEEL +5 -0
- simulo-0.26.0.dist-info/entry_points.txt +2 -0
- simulo-0.26.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
"""Content-addressed source packaging (stdlib only).
|
|
2
|
+
|
|
3
|
+
``package`` walks a project root, filters out heavy / generated files, hashes the
|
|
4
|
+
surviving tree into a stable source digest, and writes a bundle whose directory is
|
|
5
|
+
named by a short, filesystem-safe form of the *submission's* package id (see
|
|
6
|
+
:func:`bundle_dir_name`)::
|
|
7
|
+
|
|
8
|
+
<out>/sha256-<12 hex>/
|
|
9
|
+
source/ # filtered copy of the project tree
|
|
10
|
+
simulo.manifest.json # the v1 manifest (source_digest + FULL package_id stamped in)
|
|
11
|
+
|
|
12
|
+
Two identities are computed:
|
|
13
|
+
|
|
14
|
+
* ``source_digest`` — order-independent over the filtered tree (files are sorted
|
|
15
|
+
before hashing) and dependent on each file's relative path and its bytes, so
|
|
16
|
+
renames and content edits both change it. This is the *source-only* hash, kept
|
|
17
|
+
in the manifest for reproducibility / source caching.
|
|
18
|
+
* ``package_id`` — ``sha256(source_digest + "\\0" + job_name + "\\0" +
|
|
19
|
+
canonical_json(args))``. The bundle directory is named by a SHORT,
|
|
20
|
+
filesystem-safe, NON-invertible form of it (:func:`bundle_dir_name` — ``:``
|
|
21
|
+
is reserved on Windows, and a full 64-hex name is itself too long to nest
|
|
22
|
+
safely under Windows' 260-char ``MAX_PATH``; see that function's docstring).
|
|
23
|
+
Two submits of the same source with a different job or args yield a
|
|
24
|
+
different ``package_id`` (logically a distinct package), so one submission
|
|
25
|
+
never clobbers another's recorded ``args`` — the
|
|
26
|
+
"submit = immutable job" model. Submitting the *same* (source, job, args)
|
|
27
|
+
yields the *same* id (idempotent / cached) — but because the on-disk name is
|
|
28
|
+
a short, non-injective digest prefix, every consumer of a possibly-cached
|
|
29
|
+
bundle directory MUST re-verify the FULL id against the bundle's manifest
|
|
30
|
+
(:func:`read_bundle_package_id`) before trusting it as a hit; never trust
|
|
31
|
+
the directory name alone as proof of identity.
|
|
32
|
+
|
|
33
|
+
An optional deterministic tarball can be emitted alongside — named from the
|
|
34
|
+
FULL id (see :func:`tar_path_for`), so it has no truncation-collision risk.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import json
|
|
40
|
+
import os
|
|
41
|
+
import re
|
|
42
|
+
import shutil
|
|
43
|
+
import tarfile
|
|
44
|
+
from dataclasses import replace
|
|
45
|
+
from fnmatch import fnmatch
|
|
46
|
+
from hashlib import sha256
|
|
47
|
+
from pathlib import Path, PurePosixPath
|
|
48
|
+
from typing import Any, Optional
|
|
49
|
+
|
|
50
|
+
from simulo.interfaces.platform.manifest import Manifest
|
|
51
|
+
|
|
52
|
+
MANIFEST_FILENAME = "simulo.manifest.json"
|
|
53
|
+
|
|
54
|
+
# Directory names pruned wholesale (never descended into).
|
|
55
|
+
_DEFAULT_DIR_EXCLUDES = frozenset(
|
|
56
|
+
{"assets", "datasets", "checkpoints", "outputs", "logs", ".git", "__pycache__", ".simulo"}
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Glob patterns matched against each file's basename.
|
|
60
|
+
_DEFAULT_GLOB_EXCLUDES = (
|
|
61
|
+
"*.usd",
|
|
62
|
+
"*.usda",
|
|
63
|
+
"*.usdc",
|
|
64
|
+
"*.pt",
|
|
65
|
+
"*.pth",
|
|
66
|
+
"*.onnx",
|
|
67
|
+
"*.mcap",
|
|
68
|
+
"*.mp4",
|
|
69
|
+
"*.pyc",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
_SIMULOIGNORE = ".simuloignore"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _load_ignore_patterns(root: Path) -> list[str]:
|
|
76
|
+
"""Read ``.simuloignore`` (gitignore-style) from the project root, if present."""
|
|
77
|
+
ignore_file = root / _SIMULOIGNORE
|
|
78
|
+
if not ignore_file.is_file():
|
|
79
|
+
return []
|
|
80
|
+
patterns: list[str] = []
|
|
81
|
+
for raw in ignore_file.read_text(encoding="utf-8").splitlines():
|
|
82
|
+
line = raw.strip()
|
|
83
|
+
if not line or line.startswith("#"):
|
|
84
|
+
continue
|
|
85
|
+
patterns.append(line)
|
|
86
|
+
return patterns
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _matches_ignore(rel: PurePosixPath, pattern: str) -> bool:
|
|
90
|
+
"""Match one gitignore-style pattern against a POSIX relative path."""
|
|
91
|
+
rel_str = str(rel)
|
|
92
|
+
if pattern.endswith("/"):
|
|
93
|
+
directory = pattern.rstrip("/")
|
|
94
|
+
return any(part == directory for part in rel.parts) or fnmatch(rel_str, f"{directory}/*")
|
|
95
|
+
if "/" in pattern:
|
|
96
|
+
return fnmatch(rel_str, pattern)
|
|
97
|
+
return fnmatch(rel.name, pattern) or any(fnmatch(part, pattern) for part in rel.parts)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _is_excluded(rel: PurePosixPath, ignore_patterns: list[str]) -> bool:
|
|
101
|
+
if any(part in _DEFAULT_DIR_EXCLUDES for part in rel.parts):
|
|
102
|
+
return True
|
|
103
|
+
if any(fnmatch(rel.name, glob) for glob in _DEFAULT_GLOB_EXCLUDES):
|
|
104
|
+
return True
|
|
105
|
+
return any(_matches_ignore(rel, pattern) for pattern in ignore_patterns)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _collect_files(root: Path, ignore_patterns: list[str]) -> list[tuple[PurePosixPath, Path]]:
|
|
109
|
+
"""Return ``(relpath, abspath)`` for every non-excluded file under ``root``.
|
|
110
|
+
|
|
111
|
+
Symlinks (files and directories) are skipped entirely: a symlinked file whose
|
|
112
|
+
target lies outside ``root`` (e.g. ``creds -> ~/.aws/credentials``) would
|
|
113
|
+
otherwise pull out-of-tree bytes into the bundle and its digest — an
|
|
114
|
+
exfiltration vector once bundles are uploaded. ``os.walk`` already does not
|
|
115
|
+
follow symlinked directories (``followlinks=False``); we additionally prune
|
|
116
|
+
them from ``dirnames`` so they never appear as candidates.
|
|
117
|
+
"""
|
|
118
|
+
collected: list[tuple[PurePosixPath, Path]] = []
|
|
119
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
120
|
+
here = Path(dirpath)
|
|
121
|
+
# Prune excluded and symlinked directories in place so heavy / out-of-tree
|
|
122
|
+
# trees are never read.
|
|
123
|
+
kept: list[str] = []
|
|
124
|
+
for dirname in dirnames:
|
|
125
|
+
if (here / dirname).is_symlink():
|
|
126
|
+
continue
|
|
127
|
+
rel_dir = PurePosixPath((here / dirname).relative_to(root).as_posix())
|
|
128
|
+
if not _is_excluded(rel_dir, ignore_patterns):
|
|
129
|
+
kept.append(dirname)
|
|
130
|
+
dirnames[:] = kept
|
|
131
|
+
for filename in filenames:
|
|
132
|
+
abspath = here / filename
|
|
133
|
+
# Never include symlinked files — their target may lie outside ``root``.
|
|
134
|
+
if abspath.is_symlink():
|
|
135
|
+
continue
|
|
136
|
+
rel = PurePosixPath(abspath.relative_to(root).as_posix())
|
|
137
|
+
if rel.name == _SIMULOIGNORE:
|
|
138
|
+
continue
|
|
139
|
+
if _is_excluded(rel, ignore_patterns):
|
|
140
|
+
continue
|
|
141
|
+
collected.append((rel, abspath))
|
|
142
|
+
return collected
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def compute_digest(files: list[tuple[PurePosixPath, Path]]) -> str:
|
|
146
|
+
"""Compute the order-independent ``sha256:`` digest for a file set."""
|
|
147
|
+
per_file: dict[str, str] = {}
|
|
148
|
+
for rel, abspath in files:
|
|
149
|
+
rel_str = str(rel)
|
|
150
|
+
data = abspath.read_bytes()
|
|
151
|
+
per_file[rel_str] = sha256(rel_str.encode("utf-8") + b"\0" + data).hexdigest()
|
|
152
|
+
manifest_lines = "\n".join(f"{rel}:{digest}" for rel, digest in sorted(per_file.items()))
|
|
153
|
+
return "sha256:" + sha256(manifest_lines.encode("utf-8")).hexdigest()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def canonical_args(args: Optional[dict[str, Any]]) -> str:
|
|
157
|
+
"""Canonical JSON for an args dict: sorted keys, no whitespace.
|
|
158
|
+
|
|
159
|
+
The same serialisation feeds both ``compute_package_id`` and the in-process
|
|
160
|
+
bundle memo key, so two submissions with equal args map to the same string
|
|
161
|
+
(and therefore the same package id / cache slot).
|
|
162
|
+
"""
|
|
163
|
+
return json.dumps(args or {}, sort_keys=True, separators=(",", ":"))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def compute_package_id(source_digest: str, job_name: str, args: Optional[dict[str, Any]]) -> str:
|
|
167
|
+
"""Compute the submission's ``sha256:`` package id.
|
|
168
|
+
|
|
169
|
+
Identity is the *submission* — source + job + args — not the source alone, so
|
|
170
|
+
two submits of the same source with different job/args produce different ids
|
|
171
|
+
(distinct package dirs) and never clobber each other's manifest. Idempotent:
|
|
172
|
+
the same (source, job, args) yields the same id.
|
|
173
|
+
"""
|
|
174
|
+
payload = (
|
|
175
|
+
source_digest.encode("utf-8") + b"\0" + job_name.encode("utf-8") + b"\0" + canonical_args(args).encode("utf-8")
|
|
176
|
+
)
|
|
177
|
+
return "sha256:" + sha256(payload).hexdigest()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
#: The one legal shape of a package id (``compute_package_id``): the constant
|
|
181
|
+
#: ``sha256:`` prefix + exactly 64 LOWERCASE hex chars (``hexdigest()`` output).
|
|
182
|
+
_PACKAGE_ID_RE = re.compile(r"sha256:[0-9a-f]{64}")
|
|
183
|
+
|
|
184
|
+
#: Number of leading hex characters of the digest kept in the LOCAL bundle
|
|
185
|
+
#: directory name (the Docker/git short-hash convention: enough entropy to
|
|
186
|
+
#: make an accidental collision practically impossible, while keeping the
|
|
187
|
+
#: bundle path short). A full 64-hex name (``sha256-<64 hex>`` = 71 chars)
|
|
188
|
+
#: still blows Windows' 260-char ``MAX_PATH`` for a realistically nested
|
|
189
|
+
#: project under a normal-length user path — the Windows validation rig
|
|
190
|
+
#: (PR #364) measured a deepest path of 283 chars (>= 260) at a 133-char
|
|
191
|
+
#: root even AFTER the colon fix, and put the safe budget for this segment
|
|
192
|
+
#: at roughly <=35 chars. ``"sha256-" + 12 hex`` is 19 chars, well inside
|
|
193
|
+
#: that budget with headroom for deeper project trees.
|
|
194
|
+
#:
|
|
195
|
+
#: 12 hex chars = 48 bits of entropy (2**48 ≈ 2.8e14) — an accidental
|
|
196
|
+
#: same-prefix collision between two real submissions is not going to
|
|
197
|
+
#: happen in practice, but this code does NOT rely on that probability
|
|
198
|
+
#: argument alone: every read of a cached bundle directory verifies the
|
|
199
|
+
#: manifest's full ``package_id`` against the id it was asked for and
|
|
200
|
+
#: rebuilds on any mismatch (see ``read_bundle_package_id`` and
|
|
201
|
+
#: ``bundle.ensure_bundle``'s cache-hit check) — so even a genuine
|
|
202
|
+
#: collision degrades to a redundant rebuild, never a silently wrong
|
|
203
|
+
#: package (the sha256-storage-key collision class of bug the cloud-submit
|
|
204
|
+
#: wave already shipped once — see context/decisions/README.md).
|
|
205
|
+
_BUNDLE_ID_PREFIX_LEN = 12
|
|
206
|
+
|
|
207
|
+
#: Safety ceiling asserted by the MAX_PATH regression test — see
|
|
208
|
+
#: ``test_packaging.py::test_bundle_dir_name_stays_under_the_max_path_budget``.
|
|
209
|
+
#: Derived from the Windows rig's "roughly <=35 chars" finding (PR #364);
|
|
210
|
+
#: kept as an explicit constant so a future change that lengthens the name
|
|
211
|
+
#: fails a test instead of silently reintroducing WinError 123 / MAX_PATH.
|
|
212
|
+
MAX_BUNDLE_DIR_NAME_LEN = 35
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def bundle_dir_name(package_id: str) -> str:
|
|
216
|
+
"""The LOCAL filesystem directory name for a package id.
|
|
217
|
+
|
|
218
|
+
``package_id`` (``sha256:<64 hex>``) is the wire/logical identity — it goes
|
|
219
|
+
into the manifest, the submit body, and the server's storage key, and must
|
|
220
|
+
stay byte-identical everywhere off-disk. The local name differs from it in
|
|
221
|
+
two ways, both required for Windows (see ``_BUNDLE_ID_PREFIX_LEN`` for the
|
|
222
|
+
second one and its rationale):
|
|
223
|
+
|
|
224
|
+
1. ``:`` is a reserved character on Windows (the drive separator: ``mkdir``
|
|
225
|
+
raises ``WinError 123``), so the prefix is rewritten ``sha256:`` ->
|
|
226
|
+
``sha256-``.
|
|
227
|
+
2. Only the first ``_BUNDLE_ID_PREFIX_LEN`` hex characters are kept — a
|
|
228
|
+
full 64-hex name is itself long enough to blow Windows' 260-char
|
|
229
|
+
``MAX_PATH`` once nested under a realistic project tree.
|
|
230
|
+
|
|
231
|
+
Applied on every platform (not just Windows), so a project directory is
|
|
232
|
+
portable across OSes and Linux tests exercise exactly the naming that
|
|
233
|
+
ships.
|
|
234
|
+
|
|
235
|
+
**Deliberately NOT injective** — a 12-hex prefix can theoretically collide
|
|
236
|
+
between two distinct ids. This is a probability trade against Windows
|
|
237
|
+
path-length safety, not a bug: nothing in this module trusts the
|
|
238
|
+
directory name alone as proof of identity. Every read path re-verifies
|
|
239
|
+
against the bundle's manifest (``read_bundle_package_id``) and treats a
|
|
240
|
+
mismatch as a cache miss, not a hit — see ``bundle.ensure_bundle``.
|
|
241
|
+
"""
|
|
242
|
+
if not _PACKAGE_ID_RE.fullmatch(package_id):
|
|
243
|
+
raise ValueError(f"Unexpected package id shape: {package_id!r} (expected 'sha256:<64 lowercase hex>').")
|
|
244
|
+
return "sha256-" + package_id[len("sha256:") : len("sha256:") + _BUNDLE_ID_PREFIX_LEN]
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def read_bundle_package_id(bundle_dir: Path) -> Optional[str]:
|
|
248
|
+
"""Read the ``package_id`` recorded in *bundle_dir*'s manifest, if present.
|
|
249
|
+
|
|
250
|
+
Every bundle this client writes stamps ``package_id`` into its manifest —
|
|
251
|
+
``package()`` always sets it (see the two branches near the end of this
|
|
252
|
+
module), regardless of whether the caller supplied its own ``Manifest``.
|
|
253
|
+
This is the SAFE way to recover a bundle's logical id: the local directory
|
|
254
|
+
name is short and NOT invertible by design (see ``bundle_dir_name``), so
|
|
255
|
+
nothing may ever guess an id back from a directory name.
|
|
256
|
+
|
|
257
|
+
Returns ``None`` if the manifest is missing, unreadable, or carries no id
|
|
258
|
+
(a bundle written by a client older than this fix, or a manually tampered
|
|
259
|
+
/ foreign directory) — callers MUST treat that as "unknown identity" and
|
|
260
|
+
fail loudly or rebuild, never guess.
|
|
261
|
+
"""
|
|
262
|
+
manifest_path = bundle_dir / MANIFEST_FILENAME
|
|
263
|
+
if not manifest_path.is_file():
|
|
264
|
+
return None
|
|
265
|
+
try:
|
|
266
|
+
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
267
|
+
except (OSError, ValueError):
|
|
268
|
+
return None
|
|
269
|
+
package_id = data.get("package_id") if isinstance(data, dict) else None
|
|
270
|
+
return str(package_id) if package_id else None
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _write_deterministic_tar(bundle_dir: Path, tar_path: Path) -> None:
|
|
274
|
+
"""Write a reproducible tarball of ``bundle_dir`` (sorted, zeroed metadata)."""
|
|
275
|
+
members: list[tuple[str, Path]] = []
|
|
276
|
+
for path in sorted(bundle_dir.rglob("*")):
|
|
277
|
+
if path.is_file():
|
|
278
|
+
arcname = path.relative_to(bundle_dir).as_posix()
|
|
279
|
+
members.append((arcname, path))
|
|
280
|
+
|
|
281
|
+
def _reset(info: tarfile.TarInfo) -> tarfile.TarInfo:
|
|
282
|
+
info.mtime = 0
|
|
283
|
+
info.uid = info.gid = 0
|
|
284
|
+
info.uname = info.gname = ""
|
|
285
|
+
info.mode = 0o644
|
|
286
|
+
return info
|
|
287
|
+
|
|
288
|
+
with tarfile.open(tar_path, "w") as tar:
|
|
289
|
+
for arcname, path in sorted(members):
|
|
290
|
+
tar.add(path, arcname=arcname, filter=_reset)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def tar_path_for(bundle_dir: Path) -> Path:
|
|
294
|
+
"""The deterministic tar path a package's bundle directory writes to.
|
|
295
|
+
|
|
296
|
+
Named from the bundle's FULL ``package_id`` (via its manifest), not the
|
|
297
|
+
short, possibly-colliding ``bundle_dir.name`` — deliberately: unlike the
|
|
298
|
+
bundle directory itself (whose nested depth is the Windows MAX_PATH
|
|
299
|
+
concern ``bundle_dir_name`` trades against), this tar is a single sibling
|
|
300
|
+
FILE next to it, so there is no path-length reason to truncate its name,
|
|
301
|
+
and a full-length name means two DIFFERENT package ids can never produce
|
|
302
|
+
the same tar path — no truncation collision, no verify-on-read needed for
|
|
303
|
+
this one. Colon-sanitized the same way ``bundle_dir_name`` is (``:`` is
|
|
304
|
+
still illegal in a Windows filename).
|
|
305
|
+
|
|
306
|
+
Raises :class:`RuntimeError` if *bundle_dir* has no id in its manifest, OR
|
|
307
|
+
if the id present is not shape-valid: every bundle this client writes has a real
|
|
308
|
+
``sha256:<64 hex>`` id (see ``read_bundle_package_id``), so a missing OR
|
|
309
|
+
malformed id means a foreign, corrupted, or tampered directory — and
|
|
310
|
+
slicing a garbage id into a path is exactly the failure mode this module
|
|
311
|
+
refuses to have. Without the shape check, a manifest carrying e.g.
|
|
312
|
+
``"sha256:a:b"`` would reintroduce a colon into the tar path (the very
|
|
313
|
+
bug this PR exists to kill), and one carrying a too-short id could
|
|
314
|
+
collapse to a shared/degenerate path that ``ensure_tar``'s
|
|
315
|
+
reuse-without-verify would then serve for a DIFFERENT bundle.
|
|
316
|
+
"""
|
|
317
|
+
package_id = read_bundle_package_id(bundle_dir)
|
|
318
|
+
if package_id is None or not _PACKAGE_ID_RE.fullmatch(package_id):
|
|
319
|
+
raise RuntimeError(
|
|
320
|
+
f"Cannot determine a tar filename for {bundle_dir}: its manifest has no valid package_id "
|
|
321
|
+
"(the bundle predates this client version, or its manifest is missing/corrupted). "
|
|
322
|
+
"Re-run `simulo run` (or call .spawn() again) to rebuild it."
|
|
323
|
+
)
|
|
324
|
+
full_name = "sha256-" + package_id[len("sha256:") :]
|
|
325
|
+
return bundle_dir.parent / f"{full_name}.tar"
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def ensure_tar(bundle_dir: Path) -> Path:
|
|
329
|
+
"""Return the deterministic tar for *bundle_dir*, writing it if missing.
|
|
330
|
+
|
|
331
|
+
Cloud submit uploads exactly the bytes this produces (``upload_archive``);
|
|
332
|
+
written once per package id, then reused — a bundle directory's contents
|
|
333
|
+
are immutable once ``package()`` writes it (memoised by ``ensure_bundle``).
|
|
334
|
+
Safe to reuse an existing tar file without re-verifying its NAME:
|
|
335
|
+
``tar_path_for`` already ties the tar's own filename to the FULL package
|
|
336
|
+
id, so an existing tar at that exact path can only have been written for
|
|
337
|
+
this exact id (see ``tar_path_for``'s docstring).
|
|
338
|
+
|
|
339
|
+
Written to a per-process temp name first, then moved into place with
|
|
340
|
+
``os.replace`` (atomic on both POSIX and Windows) — never straight to
|
|
341
|
+
*tar_path* (a deliberate hardening). This design leans on "a file already
|
|
342
|
+
existing at this exact path == the correct bytes for this exact id"; a
|
|
343
|
+
crash or interruption mid-write with an in-place ``tarfile.open(path,
|
|
344
|
+
"w")`` would otherwise leave a truncated tar sitting at that same
|
|
345
|
+
"reuse it" path forever, silently uploaded on every future call. Any
|
|
346
|
+
write failure removes the temp file rather than leaving it stranded.
|
|
347
|
+
"""
|
|
348
|
+
tar_path = tar_path_for(bundle_dir)
|
|
349
|
+
if not tar_path.is_file():
|
|
350
|
+
tmp_path = tar_path.with_name(f"{tar_path.name}.tmp-{os.getpid()}")
|
|
351
|
+
try:
|
|
352
|
+
_write_deterministic_tar(bundle_dir, tmp_path)
|
|
353
|
+
os.replace(tmp_path, tar_path)
|
|
354
|
+
except BaseException:
|
|
355
|
+
tmp_path.unlink(missing_ok=True)
|
|
356
|
+
raise
|
|
357
|
+
return tar_path
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def package(
|
|
361
|
+
app_file: "str | Path",
|
|
362
|
+
*,
|
|
363
|
+
out_dir: "str | Path",
|
|
364
|
+
project_root: Optional["str | Path"] = None,
|
|
365
|
+
tar: bool = False,
|
|
366
|
+
manifest: Optional[Manifest] = None,
|
|
367
|
+
job_name: str = "",
|
|
368
|
+
) -> Path:
|
|
369
|
+
"""Package a project tree into a per-submission bundle.
|
|
370
|
+
|
|
371
|
+
``app_file`` is the entrypoint; ``project_root`` defaults to its directory.
|
|
372
|
+
The bundle directory is named by the *submission's* ``package_id`` —
|
|
373
|
+
``sha256(source_digest + job_name + canonical_json(args))`` — so two submits of
|
|
374
|
+
the same source with a different job or args land in distinct dirs and never
|
|
375
|
+
clobber one another. ``job_name`` names the submitted job; ``args`` are taken
|
|
376
|
+
from ``manifest.args`` (the invocation the manifest already records). A
|
|
377
|
+
``manifest`` (built by the CLI from discovered app metadata) is stamped with
|
|
378
|
+
both ``source_digest`` and ``package_id`` and written into the bundle; if
|
|
379
|
+
omitted, a minimal source-only manifest is written. Returns the bundle
|
|
380
|
+
directory.
|
|
381
|
+
|
|
382
|
+
The ``source/`` tree is written under each submission dir; duplicating source
|
|
383
|
+
across submissions of the same source is acceptable for v1 — a future
|
|
384
|
+
optimisation can dedupe by ``source_digest``.
|
|
385
|
+
"""
|
|
386
|
+
resolved_app = Path(app_file).resolve()
|
|
387
|
+
root = Path(project_root).resolve() if project_root is not None else resolved_app.parent
|
|
388
|
+
out = Path(out_dir)
|
|
389
|
+
|
|
390
|
+
if not resolved_app.is_file():
|
|
391
|
+
raise FileNotFoundError(f"App file not found: {resolved_app}")
|
|
392
|
+
if not str(resolved_app).startswith(str(root) + os.sep) and resolved_app.parent != root:
|
|
393
|
+
raise ValueError(f"App file {resolved_app} is not inside project root {root}.")
|
|
394
|
+
|
|
395
|
+
ignore_patterns = _load_ignore_patterns(root)
|
|
396
|
+
files = _collect_files(root, ignore_patterns)
|
|
397
|
+
digest = compute_digest(files)
|
|
398
|
+
|
|
399
|
+
# Identity is the submission (source + job + args), not the source alone. Args
|
|
400
|
+
# come from the manifest the caller already built with the submitted kwargs.
|
|
401
|
+
args = dict(manifest.args) if manifest is not None else {}
|
|
402
|
+
package_id = compute_package_id(digest, job_name, args)
|
|
403
|
+
|
|
404
|
+
# The directory name is the id's filesystem-safe form (never the raw id —
|
|
405
|
+
# ``:`` is illegal on Windows); the manifest/wire keep the raw ``sha256:`` id.
|
|
406
|
+
bundle_dir = out / bundle_dir_name(package_id)
|
|
407
|
+
source_dir = bundle_dir / "source"
|
|
408
|
+
if bundle_dir.exists():
|
|
409
|
+
shutil.rmtree(bundle_dir)
|
|
410
|
+
source_dir.mkdir(parents=True, exist_ok=True)
|
|
411
|
+
|
|
412
|
+
for rel, abspath in files:
|
|
413
|
+
dest = source_dir / Path(*rel.parts)
|
|
414
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
415
|
+
# ``follow_symlinks=False`` is defensive: ``_collect_files`` already drops
|
|
416
|
+
# symlinks, so ``abspath`` is a regular file here.
|
|
417
|
+
shutil.copy2(abspath, dest, follow_symlinks=False)
|
|
418
|
+
|
|
419
|
+
if manifest is None:
|
|
420
|
+
manifest = Manifest(
|
|
421
|
+
source_digest=digest,
|
|
422
|
+
runtime_id="",
|
|
423
|
+
package_id=package_id,
|
|
424
|
+
entrypoint={
|
|
425
|
+
"file": resolved_app.relative_to(root).as_posix(),
|
|
426
|
+
"module": ".".join(resolved_app.relative_to(root).with_suffix("").parts),
|
|
427
|
+
# Pre-0.15 wire spelling kept on purpose — renaming it moves
|
|
428
|
+
# archive_sha256 under a fixed package_id, the server's 409
|
|
429
|
+
# package_conflict condition. See build_manifest in bundle.py.
|
|
430
|
+
"local_entrypoint": None,
|
|
431
|
+
},
|
|
432
|
+
)
|
|
433
|
+
else:
|
|
434
|
+
manifest = replace(manifest, source_digest=digest, package_id=package_id)
|
|
435
|
+
|
|
436
|
+
(bundle_dir / MANIFEST_FILENAME).write_text(manifest.to_json(), encoding="utf-8")
|
|
437
|
+
|
|
438
|
+
if tar:
|
|
439
|
+
# Routed through ``ensure_tar`` (not a direct ``_write_deterministic_tar``
|
|
440
|
+
# call) so this gets the same atomic temp-name + ``os.replace`` write as
|
|
441
|
+
# every other tar-writing path — a crash mid-write here must never leave
|
|
442
|
+
# a truncated tar sitting at the "already written, reuse it" path.
|
|
443
|
+
ensure_tar(bundle_dir)
|
|
444
|
+
|
|
445
|
+
return bundle_dir
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""HTTP client for the pre-submit "will this work?" check.
|
|
2
|
+
|
|
3
|
+
Implements the client half of ``POST /v1/packages/preflight``
|
|
4
|
+
(:data:`simulo.interfaces.platform.manifest.PACKAGE_PREFLIGHT_ROUTE`): builds
|
|
5
|
+
the bounded request the server understands, POSTs it, and turns the response
|
|
6
|
+
into a small, render-agnostic :class:`PreflightOutcome` — ``simulo prepare``
|
|
7
|
+
(and, later, ``simulo run``'s preflight phase) decide what to do with it.
|
|
8
|
+
|
|
9
|
+
## Fail-open is the whole point (a failure class that must not be repeated)
|
|
10
|
+
|
|
11
|
+
Anything that is not a well-formed findings response — a 404 from a control
|
|
12
|
+
plane that predates this route, any 5xx, a connection failure, a transport-
|
|
13
|
+
level timeout, a malformed body — must never abort the caller. This module
|
|
14
|
+
therefore NEVER raises for those cases: :meth:`PreflightApiClient.evaluate`
|
|
15
|
+
always returns a :class:`PreflightOutcome`, either a genuine
|
|
16
|
+
``findings=(...)`` answer or ``skipped=True`` with a human-readable reason
|
|
17
|
+
for the CLI to print verbatim before proceeding. Only a genuinely
|
|
18
|
+
well-formed 200 response can carry a ``blocking`` finding.
|
|
19
|
+
|
|
20
|
+
The final ``except Exception`` in :meth:`PreflightApiClient.evaluate` is
|
|
21
|
+
deliberate, not sloppy: measured against this codebase's own ``http.py``,
|
|
22
|
+
``urllib.request.urlopen``'s connect/send-phase failures surface as
|
|
23
|
+
``urllib.error.URLError`` (wrapped here as the thin client's ``HttpUnavailable``),
|
|
24
|
+
but a peer that accepts the connection and then simply never answers (this
|
|
25
|
+
PR's own connection-timeout test reproduces it) times out INSIDE
|
|
26
|
+
``http.client.HTTPConnection.getresponse()`` — a call site ``http.py``'s
|
|
27
|
+
existing ``except urllib.error.URLError`` does not wrap, so a bare
|
|
28
|
+
``socket.timeout``/``TimeoutError`` can reach this call site unwrapped. The
|
|
29
|
+
broad catch is the safety net for exactly that gap (and for anything else
|
|
30
|
+
unanticipated) — preflight must not add a second instance of this failure class to the
|
|
31
|
+
submit path.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import json
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from typing import Any, Optional
|
|
39
|
+
|
|
40
|
+
from simulo._client import http
|
|
41
|
+
from simulo.interfaces.platform.manifest import (
|
|
42
|
+
FINDING_CLASSIFICATIONS,
|
|
43
|
+
PACKAGE_PREFLIGHT_ROUTE,
|
|
44
|
+
Manifest,
|
|
45
|
+
PreflightFinding,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
#: Every outbound call has an explicit timeout (NFR — no open-ended waits).
|
|
49
|
+
#: Module-level so tests can shrink it to force a real client-side timeout.
|
|
50
|
+
_PREFLIGHT_TIMEOUT_S = 10.0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class PreflightOutcome:
|
|
55
|
+
"""What ``simulo prepare``/``simulo run`` learned from one preflight call.
|
|
56
|
+
|
|
57
|
+
``skipped=True`` means the server never produced a well-formed findings
|
|
58
|
+
answer (see the module docstring's fail-open contract) — ``findings`` is
|
|
59
|
+
then always empty and ``notice`` is the human-readable reason the CLI
|
|
60
|
+
prints before proceeding as if preflight had not run at all.
|
|
61
|
+
``skipped=False`` means ``findings`` is the platform's real answer
|
|
62
|
+
(possibly empty — "nothing to report", a genuine pass).
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
skipped: bool
|
|
66
|
+
notice: Optional[str] = None
|
|
67
|
+
findings: tuple[PreflightFinding, ...] = field(default_factory=tuple)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _manifest_projection(manifest: Manifest) -> dict[str, Any]:
|
|
71
|
+
"""The bounded projection of *manifest* the server's request schema binds
|
|
72
|
+
(``runtime_id``, ``runtime_env``, ``jobs``, ``pip_dependencies``,
|
|
73
|
+
``manifest_version`` — see ``simulo_control_plane.preflight.schemas.
|
|
74
|
+
PreflightManifest``; every other field is ignored server-side).
|
|
75
|
+
|
|
76
|
+
Round-tripped through :meth:`Manifest.to_json` rather than hand-built
|
|
77
|
+
from the dataclass fields directly: that serialisation is already proven
|
|
78
|
+
JSON-safe (every real submit depends on it — ``jobs`` entries carry
|
|
79
|
+
values from a live ``App``'s declared job specs, not plain literals), so
|
|
80
|
+
reusing it here can never diverge from what a real submit would produce
|
|
81
|
+
for these same fields.
|
|
82
|
+
"""
|
|
83
|
+
full = json.loads(manifest.to_json())
|
|
84
|
+
return {
|
|
85
|
+
"runtime_id": full["runtime_id"],
|
|
86
|
+
"runtime_env": full["runtime_env"],
|
|
87
|
+
"jobs": full["jobs"],
|
|
88
|
+
"pip_dependencies": full["pip_dependencies"],
|
|
89
|
+
"manifest_version": full["manifest_version"],
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _notice_for_http_error(exc: http.HttpHTTPError) -> str:
|
|
94
|
+
"""Human copy for a structured non-2xx preflight response.
|
|
95
|
+
|
|
96
|
+
404 (older control plane) and 5xx get dedicated copy; everything else
|
|
97
|
+
(a 401 from an unauthenticated caller, a 422, a 429, ...) still degrades
|
|
98
|
+
to a generic skip — the fail-open contract is "anything short of a
|
|
99
|
+
well-formed findings response", not an enumerated list of statuses.
|
|
100
|
+
"""
|
|
101
|
+
if exc.status == 404:
|
|
102
|
+
return "this control plane doesn't support pre-submit checks yet (older server) — skipping"
|
|
103
|
+
if exc.status == 401:
|
|
104
|
+
return "not logged in — run `simulo login` to enable pre-submit checks; skipping for now"
|
|
105
|
+
if exc.status >= 500:
|
|
106
|
+
return f"pre-submit checks are temporarily unavailable (server error {exc.status}) — skipping"
|
|
107
|
+
return f"pre-submit checks could not run ({exc}) — skipping"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _parse_preflight_response(payload: Any) -> PreflightOutcome:
|
|
111
|
+
"""Parse a 200 body into findings, or skip if it isn't well-formed.
|
|
112
|
+
|
|
113
|
+
A malformed body (wrong shape, an unrecognised ``classification``, ...)
|
|
114
|
+
is exactly as fail-open as a 5xx — see the module docstring — never a
|
|
115
|
+
crash and never treated as "no findings" (which would look like a pass).
|
|
116
|
+
"""
|
|
117
|
+
if not isinstance(payload, dict) or not isinstance(payload.get("findings"), list):
|
|
118
|
+
return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
|
|
119
|
+
findings: list[PreflightFinding] = []
|
|
120
|
+
for raw in payload["findings"]:
|
|
121
|
+
if not isinstance(raw, dict):
|
|
122
|
+
return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
|
|
123
|
+
code = raw.get("code")
|
|
124
|
+
classification = raw.get("classification")
|
|
125
|
+
details = raw.get("details")
|
|
126
|
+
if not isinstance(code, str) or classification not in FINDING_CLASSIFICATIONS or not isinstance(details, dict):
|
|
127
|
+
return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
|
|
128
|
+
findings.append(PreflightFinding(code=code, classification=classification, details=details))
|
|
129
|
+
return PreflightOutcome(skipped=False, findings=tuple(findings))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class PreflightApiClient:
|
|
133
|
+
"""Client for ``POST /v1/packages/preflight`` — see the module docstring."""
|
|
134
|
+
|
|
135
|
+
def __init__(self, base_url: str, *, token: Optional[str] = None) -> None:
|
|
136
|
+
self._base_url = base_url.rstrip("/")
|
|
137
|
+
self._token = token
|
|
138
|
+
|
|
139
|
+
def evaluate(
|
|
140
|
+
self,
|
|
141
|
+
manifest: Manifest,
|
|
142
|
+
*,
|
|
143
|
+
job_name: Optional[str] = None,
|
|
144
|
+
archive_size: Optional[int] = None,
|
|
145
|
+
seed_from_job_id: Optional[str] = None,
|
|
146
|
+
viewstream: bool = False,
|
|
147
|
+
assets: Optional[list[dict[str, str]]] = None,
|
|
148
|
+
) -> PreflightOutcome:
|
|
149
|
+
"""Ask the platform whether this submission tuple would work.
|
|
150
|
+
|
|
151
|
+
``job_name``/``archive_size`` omitted (``simulo prepare``, which
|
|
152
|
+
checks the whole app, not one submitted job) versus both given
|
|
153
|
+
(``simulo run``'s preflight phase, PR-A6). Never raises: see the
|
|
154
|
+
module docstring's fail-open contract.
|
|
155
|
+
"""
|
|
156
|
+
body: dict[str, Any] = {"manifest": _manifest_projection(manifest)}
|
|
157
|
+
if job_name is not None:
|
|
158
|
+
body["job_name"] = job_name
|
|
159
|
+
if archive_size is not None:
|
|
160
|
+
body["archive_size"] = archive_size
|
|
161
|
+
if seed_from_job_id is not None:
|
|
162
|
+
body["seed_from_job_id"] = seed_from_job_id
|
|
163
|
+
if viewstream:
|
|
164
|
+
body["viewstream"] = True
|
|
165
|
+
if assets:
|
|
166
|
+
body["assets"] = assets
|
|
167
|
+
try:
|
|
168
|
+
payload = http.request_json(
|
|
169
|
+
"POST",
|
|
170
|
+
self._base_url + PACKAGE_PREFLIGHT_ROUTE,
|
|
171
|
+
token=self._token,
|
|
172
|
+
api_base_url=self._base_url,
|
|
173
|
+
json_body=body,
|
|
174
|
+
timeout=_PREFLIGHT_TIMEOUT_S,
|
|
175
|
+
)
|
|
176
|
+
except http.HttpHTTPError as exc:
|
|
177
|
+
return PreflightOutcome(skipped=True, notice=_notice_for_http_error(exc))
|
|
178
|
+
except http.HttpUnavailable as exc:
|
|
179
|
+
# ``exc``'s own text already reads "Cannot reach the Simulo platform at
|
|
180
|
+
# <url> (<reason>)." (``http._unavailable_message``) — don't wrap it in
|
|
181
|
+
# a second "could not reach the Simulo platform (...)" that just repeats
|
|
182
|
+
# the same phrase back to the user.
|
|
183
|
+
return PreflightOutcome(skipped=True, notice=f"{exc} — skipping")
|
|
184
|
+
except Exception as exc: # noqa: BLE001 - deliberate fail-open catch-all; see module docstring
|
|
185
|
+
return PreflightOutcome(skipped=True, notice=f"pre-submit checks failed unexpectedly ({exc}) — skipping")
|
|
186
|
+
return _parse_preflight_response(payload)
|