simulo 0.26.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. simulo/__init__.py +433 -0
  2. simulo/_client/__init__.py +6 -0
  3. simulo/_client/_entrypoint.py +313 -0
  4. simulo/_client/_mounts.py +25 -0
  5. simulo/_client/_runner.py +186 -0
  6. simulo/_client/_secure_downloads.py +1181 -0
  7. simulo/_client/app.py +1308 -0
  8. simulo/_client/asset.py +331 -0
  9. simulo/_client/asset_api.py +517 -0
  10. simulo/_client/asset_package.py +1103 -0
  11. simulo/_client/asset_pins.py +187 -0
  12. simulo/_client/builtin_aliases.py +107 -0
  13. simulo/_client/bundle.py +254 -0
  14. simulo/_client/cancel_api.py +104 -0
  15. simulo/_client/cli.py +9063 -0
  16. simulo/_client/config.py +186 -0
  17. simulo/_client/credentials.py +210 -0
  18. simulo/_client/discovery.py +214 -0
  19. simulo/_client/export_api.py +212 -0
  20. simulo/_client/export_bundle.py +296 -0
  21. simulo/_client/facades.py +581 -0
  22. simulo/_client/http.py +414 -0
  23. simulo/_client/identity_api.py +117 -0
  24. simulo/_client/install_samples.py +267 -0
  25. simulo/_client/jobs_api.py +224 -0
  26. simulo/_client/learning.py +393 -0
  27. simulo/_client/login.py +319 -0
  28. simulo/_client/mode.py +29 -0
  29. simulo/_client/outputs.py +116 -0
  30. simulo/_client/packaging.py +445 -0
  31. simulo/_client/preflight_api.py +186 -0
  32. simulo/_client/preflight_render.py +200 -0
  33. simulo/_client/registry.py +98 -0
  34. simulo/_client/runtime.py +185 -0
  35. simulo/_client/runtime_display.py +90 -0
  36. simulo/_client/seed_ref.py +76 -0
  37. simulo/_client/stub.py +41 -0
  38. simulo/_client/submit_api.py +1057 -0
  39. simulo/_client/templates/__init__.py +21 -0
  40. simulo/_client/templates/inference/app.py.tmpl +316 -0
  41. simulo/_client/templates/inference/simuloignore.tmpl +30 -0
  42. simulo/_client/templates/scenario/app.py.tmpl +93 -0
  43. simulo/_client/templates/scenario/simuloignore.tmpl +27 -0
  44. simulo/_client/templates/training/app.py.tmpl +235 -0
  45. simulo/_client/templates/training/simuloignore.tmpl +29 -0
  46. simulo/_client/view_fragment.py +21 -0
  47. simulo/_client/view_session_api.py +122 -0
  48. simulo/_client/volume.py +71 -0
  49. simulo/callbacks.py +274 -0
  50. simulo/py.typed +0 -0
  51. simulo-0.26.0.dist-info/METADATA +130 -0
  52. simulo-0.26.0.dist-info/RECORD +55 -0
  53. simulo-0.26.0.dist-info/WHEEL +5 -0
  54. simulo-0.26.0.dist-info/entry_points.txt +2 -0
  55. simulo-0.26.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,445 @@
1
+ """Content-addressed source packaging (stdlib only).
2
+
3
+ ``package`` walks a project root, filters out heavy / generated files, hashes the
4
+ surviving tree into a stable source digest, and writes a bundle whose directory is
5
+ named by a short, filesystem-safe form of the *submission's* package id (see
6
+ :func:`bundle_dir_name`)::
7
+
8
+ <out>/sha256-<12 hex>/
9
+ source/ # filtered copy of the project tree
10
+ simulo.manifest.json # the v1 manifest (source_digest + FULL package_id stamped in)
11
+
12
+ Two identities are computed:
13
+
14
+ * ``source_digest`` — order-independent over the filtered tree (files are sorted
15
+ before hashing) and dependent on each file's relative path and its bytes, so
16
+ renames and content edits both change it. This is the *source-only* hash, kept
17
+ in the manifest for reproducibility / source caching.
18
+ * ``package_id`` — ``sha256(source_digest + "\\0" + job_name + "\\0" +
19
+ canonical_json(args))``. The bundle directory is named by a SHORT,
20
+ filesystem-safe, NON-invertible form of it (:func:`bundle_dir_name` — ``:``
21
+ is reserved on Windows, and a full 64-hex name is itself too long to nest
22
+ safely under Windows' 260-char ``MAX_PATH``; see that function's docstring).
23
+ Two submits of the same source with a different job or args yield a
24
+ different ``package_id`` (logically a distinct package), so one submission
25
+ never clobbers another's recorded ``args`` — the
26
+ "submit = immutable job" model. Submitting the *same* (source, job, args)
27
+ yields the *same* id (idempotent / cached) — but because the on-disk name is
28
+ a short, non-injective digest prefix, every consumer of a possibly-cached
29
+ bundle directory MUST re-verify the FULL id against the bundle's manifest
30
+ (:func:`read_bundle_package_id`) before trusting it as a hit; never trust
31
+ the directory name alone as proof of identity.
32
+
33
+ An optional deterministic tarball can be emitted alongside — named from the
34
+ FULL id (see :func:`tar_path_for`), so it has no truncation-collision risk.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import json
40
+ import os
41
+ import re
42
+ import shutil
43
+ import tarfile
44
+ from dataclasses import replace
45
+ from fnmatch import fnmatch
46
+ from hashlib import sha256
47
+ from pathlib import Path, PurePosixPath
48
+ from typing import Any, Optional
49
+
50
+ from simulo.interfaces.platform.manifest import Manifest
51
+
52
+ MANIFEST_FILENAME = "simulo.manifest.json"
53
+
54
+ # Directory names pruned wholesale (never descended into).
55
+ _DEFAULT_DIR_EXCLUDES = frozenset(
56
+ {"assets", "datasets", "checkpoints", "outputs", "logs", ".git", "__pycache__", ".simulo"}
57
+ )
58
+
59
+ # Glob patterns matched against each file's basename.
60
+ _DEFAULT_GLOB_EXCLUDES = (
61
+ "*.usd",
62
+ "*.usda",
63
+ "*.usdc",
64
+ "*.pt",
65
+ "*.pth",
66
+ "*.onnx",
67
+ "*.mcap",
68
+ "*.mp4",
69
+ "*.pyc",
70
+ )
71
+
72
+ _SIMULOIGNORE = ".simuloignore"
73
+
74
+
75
+ def _load_ignore_patterns(root: Path) -> list[str]:
76
+ """Read ``.simuloignore`` (gitignore-style) from the project root, if present."""
77
+ ignore_file = root / _SIMULOIGNORE
78
+ if not ignore_file.is_file():
79
+ return []
80
+ patterns: list[str] = []
81
+ for raw in ignore_file.read_text(encoding="utf-8").splitlines():
82
+ line = raw.strip()
83
+ if not line or line.startswith("#"):
84
+ continue
85
+ patterns.append(line)
86
+ return patterns
87
+
88
+
89
+ def _matches_ignore(rel: PurePosixPath, pattern: str) -> bool:
90
+ """Match one gitignore-style pattern against a POSIX relative path."""
91
+ rel_str = str(rel)
92
+ if pattern.endswith("/"):
93
+ directory = pattern.rstrip("/")
94
+ return any(part == directory for part in rel.parts) or fnmatch(rel_str, f"{directory}/*")
95
+ if "/" in pattern:
96
+ return fnmatch(rel_str, pattern)
97
+ return fnmatch(rel.name, pattern) or any(fnmatch(part, pattern) for part in rel.parts)
98
+
99
+
100
+ def _is_excluded(rel: PurePosixPath, ignore_patterns: list[str]) -> bool:
101
+ if any(part in _DEFAULT_DIR_EXCLUDES for part in rel.parts):
102
+ return True
103
+ if any(fnmatch(rel.name, glob) for glob in _DEFAULT_GLOB_EXCLUDES):
104
+ return True
105
+ return any(_matches_ignore(rel, pattern) for pattern in ignore_patterns)
106
+
107
+
108
+ def _collect_files(root: Path, ignore_patterns: list[str]) -> list[tuple[PurePosixPath, Path]]:
109
+ """Return ``(relpath, abspath)`` for every non-excluded file under ``root``.
110
+
111
+ Symlinks (files and directories) are skipped entirely: a symlinked file whose
112
+ target lies outside ``root`` (e.g. ``creds -> ~/.aws/credentials``) would
113
+ otherwise pull out-of-tree bytes into the bundle and its digest — an
114
+ exfiltration vector once bundles are uploaded. ``os.walk`` already does not
115
+ follow symlinked directories (``followlinks=False``); we additionally prune
116
+ them from ``dirnames`` so they never appear as candidates.
117
+ """
118
+ collected: list[tuple[PurePosixPath, Path]] = []
119
+ for dirpath, dirnames, filenames in os.walk(root):
120
+ here = Path(dirpath)
121
+ # Prune excluded and symlinked directories in place so heavy / out-of-tree
122
+ # trees are never read.
123
+ kept: list[str] = []
124
+ for dirname in dirnames:
125
+ if (here / dirname).is_symlink():
126
+ continue
127
+ rel_dir = PurePosixPath((here / dirname).relative_to(root).as_posix())
128
+ if not _is_excluded(rel_dir, ignore_patterns):
129
+ kept.append(dirname)
130
+ dirnames[:] = kept
131
+ for filename in filenames:
132
+ abspath = here / filename
133
+ # Never include symlinked files — their target may lie outside ``root``.
134
+ if abspath.is_symlink():
135
+ continue
136
+ rel = PurePosixPath(abspath.relative_to(root).as_posix())
137
+ if rel.name == _SIMULOIGNORE:
138
+ continue
139
+ if _is_excluded(rel, ignore_patterns):
140
+ continue
141
+ collected.append((rel, abspath))
142
+ return collected
143
+
144
+
145
+ def compute_digest(files: list[tuple[PurePosixPath, Path]]) -> str:
146
+ """Compute the order-independent ``sha256:`` digest for a file set."""
147
+ per_file: dict[str, str] = {}
148
+ for rel, abspath in files:
149
+ rel_str = str(rel)
150
+ data = abspath.read_bytes()
151
+ per_file[rel_str] = sha256(rel_str.encode("utf-8") + b"\0" + data).hexdigest()
152
+ manifest_lines = "\n".join(f"{rel}:{digest}" for rel, digest in sorted(per_file.items()))
153
+ return "sha256:" + sha256(manifest_lines.encode("utf-8")).hexdigest()
154
+
155
+
156
+ def canonical_args(args: Optional[dict[str, Any]]) -> str:
157
+ """Canonical JSON for an args dict: sorted keys, no whitespace.
158
+
159
+ The same serialisation feeds both ``compute_package_id`` and the in-process
160
+ bundle memo key, so two submissions with equal args map to the same string
161
+ (and therefore the same package id / cache slot).
162
+ """
163
+ return json.dumps(args or {}, sort_keys=True, separators=(",", ":"))
164
+
165
+
166
+ def compute_package_id(source_digest: str, job_name: str, args: Optional[dict[str, Any]]) -> str:
167
+ """Compute the submission's ``sha256:`` package id.
168
+
169
+ Identity is the *submission* — source + job + args — not the source alone, so
170
+ two submits of the same source with different job/args produce different ids
171
+ (distinct package dirs) and never clobber each other's manifest. Idempotent:
172
+ the same (source, job, args) yields the same id.
173
+ """
174
+ payload = (
175
+ source_digest.encode("utf-8") + b"\0" + job_name.encode("utf-8") + b"\0" + canonical_args(args).encode("utf-8")
176
+ )
177
+ return "sha256:" + sha256(payload).hexdigest()
178
+
179
+
180
+ #: The one legal shape of a package id (``compute_package_id``): the constant
181
+ #: ``sha256:`` prefix + exactly 64 LOWERCASE hex chars (``hexdigest()`` output).
182
+ _PACKAGE_ID_RE = re.compile(r"sha256:[0-9a-f]{64}")
183
+
184
+ #: Number of leading hex characters of the digest kept in the LOCAL bundle
185
+ #: directory name (the Docker/git short-hash convention: enough entropy to
186
+ #: make an accidental collision practically impossible, while keeping the
187
+ #: bundle path short). A full 64-hex name (``sha256-<64 hex>`` = 71 chars)
188
+ #: still blows Windows' 260-char ``MAX_PATH`` for a realistically nested
189
+ #: project under a normal-length user path — the Windows validation rig
190
+ #: (PR #364) measured a deepest path of 283 chars (>= 260) at a 133-char
191
+ #: root even AFTER the colon fix, and put the safe budget for this segment
192
+ #: at roughly <=35 chars. ``"sha256-" + 12 hex`` is 19 chars, well inside
193
+ #: that budget with headroom for deeper project trees.
194
+ #:
195
+ #: 12 hex chars = 48 bits of entropy (2**48 ≈ 2.8e14) — an accidental
196
+ #: same-prefix collision between two real submissions is not going to
197
+ #: happen in practice, but this code does NOT rely on that probability
198
+ #: argument alone: every read of a cached bundle directory verifies the
199
+ #: manifest's full ``package_id`` against the id it was asked for and
200
+ #: rebuilds on any mismatch (see ``read_bundle_package_id`` and
201
+ #: ``bundle.ensure_bundle``'s cache-hit check) — so even a genuine
202
+ #: collision degrades to a redundant rebuild, never a silently wrong
203
+ #: package (the sha256-storage-key collision class of bug the cloud-submit
204
+ #: wave already shipped once — see context/decisions/README.md).
205
+ _BUNDLE_ID_PREFIX_LEN = 12
206
+
207
+ #: Safety ceiling asserted by the MAX_PATH regression test — see
208
+ #: ``test_packaging.py::test_bundle_dir_name_stays_under_the_max_path_budget``.
209
+ #: Derived from the Windows rig's "roughly <=35 chars" finding (PR #364);
210
+ #: kept as an explicit constant so a future change that lengthens the name
211
+ #: fails a test instead of silently reintroducing WinError 123 / MAX_PATH.
212
+ MAX_BUNDLE_DIR_NAME_LEN = 35
213
+
214
+
215
+ def bundle_dir_name(package_id: str) -> str:
216
+ """The LOCAL filesystem directory name for a package id.
217
+
218
+ ``package_id`` (``sha256:<64 hex>``) is the wire/logical identity — it goes
219
+ into the manifest, the submit body, and the server's storage key, and must
220
+ stay byte-identical everywhere off-disk. The local name differs from it in
221
+ two ways, both required for Windows (see ``_BUNDLE_ID_PREFIX_LEN`` for the
222
+ second one and its rationale):
223
+
224
+ 1. ``:`` is a reserved character on Windows (the drive separator: ``mkdir``
225
+ raises ``WinError 123``), so the prefix is rewritten ``sha256:`` ->
226
+ ``sha256-``.
227
+ 2. Only the first ``_BUNDLE_ID_PREFIX_LEN`` hex characters are kept — a
228
+ full 64-hex name is itself long enough to blow Windows' 260-char
229
+ ``MAX_PATH`` once nested under a realistic project tree.
230
+
231
+ Applied on every platform (not just Windows), so a project directory is
232
+ portable across OSes and Linux tests exercise exactly the naming that
233
+ ships.
234
+
235
+ **Deliberately NOT injective** — a 12-hex prefix can theoretically collide
236
+ between two distinct ids. This is a probability trade against Windows
237
+ path-length safety, not a bug: nothing in this module trusts the
238
+ directory name alone as proof of identity. Every read path re-verifies
239
+ against the bundle's manifest (``read_bundle_package_id``) and treats a
240
+ mismatch as a cache miss, not a hit — see ``bundle.ensure_bundle``.
241
+ """
242
+ if not _PACKAGE_ID_RE.fullmatch(package_id):
243
+ raise ValueError(f"Unexpected package id shape: {package_id!r} (expected 'sha256:<64 lowercase hex>').")
244
+ return "sha256-" + package_id[len("sha256:") : len("sha256:") + _BUNDLE_ID_PREFIX_LEN]
245
+
246
+
247
+ def read_bundle_package_id(bundle_dir: Path) -> Optional[str]:
248
+ """Read the ``package_id`` recorded in *bundle_dir*'s manifest, if present.
249
+
250
+ Every bundle this client writes stamps ``package_id`` into its manifest —
251
+ ``package()`` always sets it (see the two branches near the end of this
252
+ module), regardless of whether the caller supplied its own ``Manifest``.
253
+ This is the SAFE way to recover a bundle's logical id: the local directory
254
+ name is short and NOT invertible by design (see ``bundle_dir_name``), so
255
+ nothing may ever guess an id back from a directory name.
256
+
257
+ Returns ``None`` if the manifest is missing, unreadable, or carries no id
258
+ (a bundle written by a client older than this fix, or a manually tampered
259
+ / foreign directory) — callers MUST treat that as "unknown identity" and
260
+ fail loudly or rebuild, never guess.
261
+ """
262
+ manifest_path = bundle_dir / MANIFEST_FILENAME
263
+ if not manifest_path.is_file():
264
+ return None
265
+ try:
266
+ data = json.loads(manifest_path.read_text(encoding="utf-8"))
267
+ except (OSError, ValueError):
268
+ return None
269
+ package_id = data.get("package_id") if isinstance(data, dict) else None
270
+ return str(package_id) if package_id else None
271
+
272
+
273
+ def _write_deterministic_tar(bundle_dir: Path, tar_path: Path) -> None:
274
+ """Write a reproducible tarball of ``bundle_dir`` (sorted, zeroed metadata)."""
275
+ members: list[tuple[str, Path]] = []
276
+ for path in sorted(bundle_dir.rglob("*")):
277
+ if path.is_file():
278
+ arcname = path.relative_to(bundle_dir).as_posix()
279
+ members.append((arcname, path))
280
+
281
+ def _reset(info: tarfile.TarInfo) -> tarfile.TarInfo:
282
+ info.mtime = 0
283
+ info.uid = info.gid = 0
284
+ info.uname = info.gname = ""
285
+ info.mode = 0o644
286
+ return info
287
+
288
+ with tarfile.open(tar_path, "w") as tar:
289
+ for arcname, path in sorted(members):
290
+ tar.add(path, arcname=arcname, filter=_reset)
291
+
292
+
293
+ def tar_path_for(bundle_dir: Path) -> Path:
294
+ """The deterministic tar path a package's bundle directory writes to.
295
+
296
+ Named from the bundle's FULL ``package_id`` (via its manifest), not the
297
+ short, possibly-colliding ``bundle_dir.name`` — deliberately: unlike the
298
+ bundle directory itself (whose nested depth is the Windows MAX_PATH
299
+ concern ``bundle_dir_name`` trades against), this tar is a single sibling
300
+ FILE next to it, so there is no path-length reason to truncate its name,
301
+ and a full-length name means two DIFFERENT package ids can never produce
302
+ the same tar path — no truncation collision, no verify-on-read needed for
303
+ this one. Colon-sanitized the same way ``bundle_dir_name`` is (``:`` is
304
+ still illegal in a Windows filename).
305
+
306
+ Raises :class:`RuntimeError` if *bundle_dir* has no id in its manifest, OR
307
+ if the id present is not shape-valid: every bundle this client writes has a real
308
+ ``sha256:<64 hex>`` id (see ``read_bundle_package_id``), so a missing OR
309
+ malformed id means a foreign, corrupted, or tampered directory — and
310
+ slicing a garbage id into a path is exactly the failure mode this module
311
+ refuses to have. Without the shape check, a manifest carrying e.g.
312
+ ``"sha256:a:b"`` would reintroduce a colon into the tar path (the very
313
+ bug this PR exists to kill), and one carrying a too-short id could
314
+ collapse to a shared/degenerate path that ``ensure_tar``'s
315
+ reuse-without-verify would then serve for a DIFFERENT bundle.
316
+ """
317
+ package_id = read_bundle_package_id(bundle_dir)
318
+ if package_id is None or not _PACKAGE_ID_RE.fullmatch(package_id):
319
+ raise RuntimeError(
320
+ f"Cannot determine a tar filename for {bundle_dir}: its manifest has no valid package_id "
321
+ "(the bundle predates this client version, or its manifest is missing/corrupted). "
322
+ "Re-run `simulo run` (or call .spawn() again) to rebuild it."
323
+ )
324
+ full_name = "sha256-" + package_id[len("sha256:") :]
325
+ return bundle_dir.parent / f"{full_name}.tar"
326
+
327
+
328
+ def ensure_tar(bundle_dir: Path) -> Path:
329
+ """Return the deterministic tar for *bundle_dir*, writing it if missing.
330
+
331
+ Cloud submit uploads exactly the bytes this produces (``upload_archive``);
332
+ written once per package id, then reused — a bundle directory's contents
333
+ are immutable once ``package()`` writes it (memoised by ``ensure_bundle``).
334
+ Safe to reuse an existing tar file without re-verifying its NAME:
335
+ ``tar_path_for`` already ties the tar's own filename to the FULL package
336
+ id, so an existing tar at that exact path can only have been written for
337
+ this exact id (see ``tar_path_for``'s docstring).
338
+
339
+ Written to a per-process temp name first, then moved into place with
340
+ ``os.replace`` (atomic on both POSIX and Windows) — never straight to
341
+ *tar_path* (a deliberate hardening). This design leans on "a file already
342
+ existing at this exact path == the correct bytes for this exact id"; a
343
+ crash or interruption mid-write with an in-place ``tarfile.open(path,
344
+ "w")`` would otherwise leave a truncated tar sitting at that same
345
+ "reuse it" path forever, silently uploaded on every future call. Any
346
+ write failure removes the temp file rather than leaving it stranded.
347
+ """
348
+ tar_path = tar_path_for(bundle_dir)
349
+ if not tar_path.is_file():
350
+ tmp_path = tar_path.with_name(f"{tar_path.name}.tmp-{os.getpid()}")
351
+ try:
352
+ _write_deterministic_tar(bundle_dir, tmp_path)
353
+ os.replace(tmp_path, tar_path)
354
+ except BaseException:
355
+ tmp_path.unlink(missing_ok=True)
356
+ raise
357
+ return tar_path
358
+
359
+
360
+ def package(
361
+ app_file: "str | Path",
362
+ *,
363
+ out_dir: "str | Path",
364
+ project_root: Optional["str | Path"] = None,
365
+ tar: bool = False,
366
+ manifest: Optional[Manifest] = None,
367
+ job_name: str = "",
368
+ ) -> Path:
369
+ """Package a project tree into a per-submission bundle.
370
+
371
+ ``app_file`` is the entrypoint; ``project_root`` defaults to its directory.
372
+ The bundle directory is named by the *submission's* ``package_id`` —
373
+ ``sha256(source_digest + job_name + canonical_json(args))`` — so two submits of
374
+ the same source with a different job or args land in distinct dirs and never
375
+ clobber one another. ``job_name`` names the submitted job; ``args`` are taken
376
+ from ``manifest.args`` (the invocation the manifest already records). A
377
+ ``manifest`` (built by the CLI from discovered app metadata) is stamped with
378
+ both ``source_digest`` and ``package_id`` and written into the bundle; if
379
+ omitted, a minimal source-only manifest is written. Returns the bundle
380
+ directory.
381
+
382
+ The ``source/`` tree is written under each submission dir; duplicating source
383
+ across submissions of the same source is acceptable for v1 — a future
384
+ optimisation can dedupe by ``source_digest``.
385
+ """
386
+ resolved_app = Path(app_file).resolve()
387
+ root = Path(project_root).resolve() if project_root is not None else resolved_app.parent
388
+ out = Path(out_dir)
389
+
390
+ if not resolved_app.is_file():
391
+ raise FileNotFoundError(f"App file not found: {resolved_app}")
392
+ if not str(resolved_app).startswith(str(root) + os.sep) and resolved_app.parent != root:
393
+ raise ValueError(f"App file {resolved_app} is not inside project root {root}.")
394
+
395
+ ignore_patterns = _load_ignore_patterns(root)
396
+ files = _collect_files(root, ignore_patterns)
397
+ digest = compute_digest(files)
398
+
399
+ # Identity is the submission (source + job + args), not the source alone. Args
400
+ # come from the manifest the caller already built with the submitted kwargs.
401
+ args = dict(manifest.args) if manifest is not None else {}
402
+ package_id = compute_package_id(digest, job_name, args)
403
+
404
+ # The directory name is the id's filesystem-safe form (never the raw id —
405
+ # ``:`` is illegal on Windows); the manifest/wire keep the raw ``sha256:`` id.
406
+ bundle_dir = out / bundle_dir_name(package_id)
407
+ source_dir = bundle_dir / "source"
408
+ if bundle_dir.exists():
409
+ shutil.rmtree(bundle_dir)
410
+ source_dir.mkdir(parents=True, exist_ok=True)
411
+
412
+ for rel, abspath in files:
413
+ dest = source_dir / Path(*rel.parts)
414
+ dest.parent.mkdir(parents=True, exist_ok=True)
415
+ # ``follow_symlinks=False`` is defensive: ``_collect_files`` already drops
416
+ # symlinks, so ``abspath`` is a regular file here.
417
+ shutil.copy2(abspath, dest, follow_symlinks=False)
418
+
419
+ if manifest is None:
420
+ manifest = Manifest(
421
+ source_digest=digest,
422
+ runtime_id="",
423
+ package_id=package_id,
424
+ entrypoint={
425
+ "file": resolved_app.relative_to(root).as_posix(),
426
+ "module": ".".join(resolved_app.relative_to(root).with_suffix("").parts),
427
+ # Pre-0.15 wire spelling kept on purpose — renaming it moves
428
+ # archive_sha256 under a fixed package_id, the server's 409
429
+ # package_conflict condition. See build_manifest in bundle.py.
430
+ "local_entrypoint": None,
431
+ },
432
+ )
433
+ else:
434
+ manifest = replace(manifest, source_digest=digest, package_id=package_id)
435
+
436
+ (bundle_dir / MANIFEST_FILENAME).write_text(manifest.to_json(), encoding="utf-8")
437
+
438
+ if tar:
439
+ # Routed through ``ensure_tar`` (not a direct ``_write_deterministic_tar``
440
+ # call) so this gets the same atomic temp-name + ``os.replace`` write as
441
+ # every other tar-writing path — a crash mid-write here must never leave
442
+ # a truncated tar sitting at the "already written, reuse it" path.
443
+ ensure_tar(bundle_dir)
444
+
445
+ return bundle_dir
@@ -0,0 +1,186 @@
1
+ """HTTP client for the pre-submit "will this work?" check.
2
+
3
+ Implements the client half of ``POST /v1/packages/preflight``
4
+ (:data:`simulo.interfaces.platform.manifest.PACKAGE_PREFLIGHT_ROUTE`): builds
5
+ the bounded request the server understands, POSTs it, and turns the response
6
+ into a small, render-agnostic :class:`PreflightOutcome` — ``simulo prepare``
7
+ (and, later, ``simulo run``'s preflight phase) decide what to do with it.
8
+
9
+ ## Fail-open is the whole point (a failure class that must not be repeated)
10
+
11
+ Anything that is not a well-formed findings response — a 404 from a control
12
+ plane that predates this route, any 5xx, a connection failure, a transport-
13
+ level timeout, a malformed body — must never abort the caller. This module
14
+ therefore NEVER raises for those cases: :meth:`PreflightApiClient.evaluate`
15
+ always returns a :class:`PreflightOutcome`, either a genuine
16
+ ``findings=(...)`` answer or ``skipped=True`` with a human-readable reason
17
+ for the CLI to print verbatim before proceeding. Only a genuinely
18
+ well-formed 200 response can carry a ``blocking`` finding.
19
+
20
+ The final ``except Exception`` in :meth:`PreflightApiClient.evaluate` is
21
+ deliberate, not sloppy: measured against this codebase's own ``http.py``,
22
+ ``urllib.request.urlopen``'s connect/send-phase failures surface as
23
+ ``urllib.error.URLError`` (wrapped here as the thin client's ``HttpUnavailable``),
24
+ but a peer that accepts the connection and then simply never answers (this
25
+ PR's own connection-timeout test reproduces it) times out INSIDE
26
+ ``http.client.HTTPConnection.getresponse()`` — a call site ``http.py``'s
27
+ existing ``except urllib.error.URLError`` does not wrap, so a bare
28
+ ``socket.timeout``/``TimeoutError`` can reach this call site unwrapped. The
29
+ broad catch is the safety net for exactly that gap (and for anything else
30
+ unanticipated) — preflight must not add a second instance of this failure class to the
31
+ submit path.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ from dataclasses import dataclass, field
38
+ from typing import Any, Optional
39
+
40
+ from simulo._client import http
41
+ from simulo.interfaces.platform.manifest import (
42
+ FINDING_CLASSIFICATIONS,
43
+ PACKAGE_PREFLIGHT_ROUTE,
44
+ Manifest,
45
+ PreflightFinding,
46
+ )
47
+
48
+ #: Every outbound call has an explicit timeout (NFR — no open-ended waits).
49
+ #: Module-level so tests can shrink it to force a real client-side timeout.
50
+ _PREFLIGHT_TIMEOUT_S = 10.0
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class PreflightOutcome:
55
+ """What ``simulo prepare``/``simulo run`` learned from one preflight call.
56
+
57
+ ``skipped=True`` means the server never produced a well-formed findings
58
+ answer (see the module docstring's fail-open contract) — ``findings`` is
59
+ then always empty and ``notice`` is the human-readable reason the CLI
60
+ prints before proceeding as if preflight had not run at all.
61
+ ``skipped=False`` means ``findings`` is the platform's real answer
62
+ (possibly empty — "nothing to report", a genuine pass).
63
+ """
64
+
65
+ skipped: bool
66
+ notice: Optional[str] = None
67
+ findings: tuple[PreflightFinding, ...] = field(default_factory=tuple)
68
+
69
+
70
+ def _manifest_projection(manifest: Manifest) -> dict[str, Any]:
71
+ """The bounded projection of *manifest* the server's request schema binds
72
+ (``runtime_id``, ``runtime_env``, ``jobs``, ``pip_dependencies``,
73
+ ``manifest_version`` — see ``simulo_control_plane.preflight.schemas.
74
+ PreflightManifest``; every other field is ignored server-side).
75
+
76
+ Round-tripped through :meth:`Manifest.to_json` rather than hand-built
77
+ from the dataclass fields directly: that serialisation is already proven
78
+ JSON-safe (every real submit depends on it — ``jobs`` entries carry
79
+ values from a live ``App``'s declared job specs, not plain literals), so
80
+ reusing it here can never diverge from what a real submit would produce
81
+ for these same fields.
82
+ """
83
+ full = json.loads(manifest.to_json())
84
+ return {
85
+ "runtime_id": full["runtime_id"],
86
+ "runtime_env": full["runtime_env"],
87
+ "jobs": full["jobs"],
88
+ "pip_dependencies": full["pip_dependencies"],
89
+ "manifest_version": full["manifest_version"],
90
+ }
91
+
92
+
93
+ def _notice_for_http_error(exc: http.HttpHTTPError) -> str:
94
+ """Human copy for a structured non-2xx preflight response.
95
+
96
+ 404 (older control plane) and 5xx get dedicated copy; everything else
97
+ (a 401 from an unauthenticated caller, a 422, a 429, ...) still degrades
98
+ to a generic skip — the fail-open contract is "anything short of a
99
+ well-formed findings response", not an enumerated list of statuses.
100
+ """
101
+ if exc.status == 404:
102
+ return "this control plane doesn't support pre-submit checks yet (older server) — skipping"
103
+ if exc.status == 401:
104
+ return "not logged in — run `simulo login` to enable pre-submit checks; skipping for now"
105
+ if exc.status >= 500:
106
+ return f"pre-submit checks are temporarily unavailable (server error {exc.status}) — skipping"
107
+ return f"pre-submit checks could not run ({exc}) — skipping"
108
+
109
+
110
+ def _parse_preflight_response(payload: Any) -> PreflightOutcome:
111
+ """Parse a 200 body into findings, or skip if it isn't well-formed.
112
+
113
+ A malformed body (wrong shape, an unrecognised ``classification``, ...)
114
+ is exactly as fail-open as a 5xx — see the module docstring — never a
115
+ crash and never treated as "no findings" (which would look like a pass).
116
+ """
117
+ if not isinstance(payload, dict) or not isinstance(payload.get("findings"), list):
118
+ return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
119
+ findings: list[PreflightFinding] = []
120
+ for raw in payload["findings"]:
121
+ if not isinstance(raw, dict):
122
+ return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
123
+ code = raw.get("code")
124
+ classification = raw.get("classification")
125
+ details = raw.get("details")
126
+ if not isinstance(code, str) or classification not in FINDING_CLASSIFICATIONS or not isinstance(details, dict):
127
+ return PreflightOutcome(skipped=True, notice="the preflight response was malformed — skipping")
128
+ findings.append(PreflightFinding(code=code, classification=classification, details=details))
129
+ return PreflightOutcome(skipped=False, findings=tuple(findings))
130
+
131
+
132
+ class PreflightApiClient:
133
+ """Client for ``POST /v1/packages/preflight`` — see the module docstring."""
134
+
135
+ def __init__(self, base_url: str, *, token: Optional[str] = None) -> None:
136
+ self._base_url = base_url.rstrip("/")
137
+ self._token = token
138
+
139
+ def evaluate(
140
+ self,
141
+ manifest: Manifest,
142
+ *,
143
+ job_name: Optional[str] = None,
144
+ archive_size: Optional[int] = None,
145
+ seed_from_job_id: Optional[str] = None,
146
+ viewstream: bool = False,
147
+ assets: Optional[list[dict[str, str]]] = None,
148
+ ) -> PreflightOutcome:
149
+ """Ask the platform whether this submission tuple would work.
150
+
151
+ ``job_name``/``archive_size`` omitted (``simulo prepare``, which
152
+ checks the whole app, not one submitted job) versus both given
153
+ (``simulo run``'s preflight phase, PR-A6). Never raises: see the
154
+ module docstring's fail-open contract.
155
+ """
156
+ body: dict[str, Any] = {"manifest": _manifest_projection(manifest)}
157
+ if job_name is not None:
158
+ body["job_name"] = job_name
159
+ if archive_size is not None:
160
+ body["archive_size"] = archive_size
161
+ if seed_from_job_id is not None:
162
+ body["seed_from_job_id"] = seed_from_job_id
163
+ if viewstream:
164
+ body["viewstream"] = True
165
+ if assets:
166
+ body["assets"] = assets
167
+ try:
168
+ payload = http.request_json(
169
+ "POST",
170
+ self._base_url + PACKAGE_PREFLIGHT_ROUTE,
171
+ token=self._token,
172
+ api_base_url=self._base_url,
173
+ json_body=body,
174
+ timeout=_PREFLIGHT_TIMEOUT_S,
175
+ )
176
+ except http.HttpHTTPError as exc:
177
+ return PreflightOutcome(skipped=True, notice=_notice_for_http_error(exc))
178
+ except http.HttpUnavailable as exc:
179
+ # ``exc``'s own text already reads "Cannot reach the Simulo platform at
180
+ # <url> (<reason>)." (``http._unavailable_message``) — don't wrap it in
181
+ # a second "could not reach the Simulo platform (...)" that just repeats
182
+ # the same phrase back to the user.
183
+ return PreflightOutcome(skipped=True, notice=f"{exc} — skipping")
184
+ except Exception as exc: # noqa: BLE001 - deliberate fail-open catch-all; see module docstring
185
+ return PreflightOutcome(skipped=True, notice=f"pre-submit checks failed unexpectedly ({exc}) — skipping")
186
+ return _parse_preflight_response(payload)