simulo 0.26.0__py3-none-any.whl

simulo/_client/submit_api.py

@@ -0,0 +1,1057 @@
+"""HTTP client for the cloud submit + recordings + models surface (stdlib only, torch-free).
+
+Implements the client-facing half of ``simulo.interfaces.platform.submit``:
+register a content-addressed package, upload its tar archive, create the job
+that executes it, and list/download the MCAP recordings and trained-model
+checkpoints a worker produced. Shares request plumbing (structured errors,
+the https-when-token guard, the foreign-host bearer rule) with
+``jobs_api.py`` via ``http.py``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import urllib.parse
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Optional
+from uuid import UUID
+
+from simulo._client import http
+from simulo._client._secure_downloads import (
+    DestinationDirectoryError,
+    DestinationObservation,
+    observe_destination,
+    open_destination_directory,
+)
+from simulo._client.preflight_render import sanitize_server_text
+from simulo.interfaces.ids import parse_job_public_id
+from simulo.interfaces.platform.asset_catalog import (
+    ASSETS_FIELD,
+    ASSETS_RESOLVE_ROUTE,
+    MAX_ASSETS_RESOLVE_BATCH,
+    AssetsResolveResponse,
+    ResolvedAssetPin,
+    ResolveMiss,
+)
+from simulo.interfaces.platform.outputs import (
+    JOB_OUTPUT_DOWNLOAD_ROUTE_TEMPLATE,
+    JOB_OUTPUTS_ROUTE_TEMPLATE,
+    OutputRecord,
+)
+from simulo.interfaces.platform.runs import JOB_SCOPE_MINE, JOBS_ROUTE
+from simulo.interfaces.platform.submit import (
+    JOB_MODEL_DOWNLOAD_ROUTE_TEMPLATE,
+    JOB_MODEL_ROUTE_TEMPLATE,
+    JOB_MODELS_ROUTE_TEMPLATE,
+    JOB_RECORDING_DOWNLOAD_ROUTE_TEMPLATE,
+    JOB_RECORDINGS_ROUTE_TEMPLATE,
+    PACKAGE_ARCHIVE_ROUTE_TEMPLATE,
+    PACKAGES_ROUTE,
+    SEED_SOURCE_CONFLICT_CODE,
+    SEED_SOURCE_FIELDS,
+)
+
+_REQUEST_TIMEOUT_S = 10.0  # every outbound call has an explicit timeout (NFR)
+_UPLOAD_TIMEOUT_S = 60.0  # package archives can be up to 64 MiB (MAX_PACKAGE_BYTES)
+_DOWNLOAD_TIMEOUT_S = 60.0
+
+_LIST_PAGE_LIMIT = 100
+_LIST_MAX_PAGES = 1000
+
+_UNAVAILABLE_HINT = "Check SIMULO_API_URL / SIMULO_ENV, and that you are logged in (`simulo login`)."
+
+#: Width cap on any server-derived value echoed back in an error message here
+#: (see :func:`_safe_server_text`). A real sha256
+#: hex digest is 64 chars and a real download destination is a filename under
+#: the user's own cwd, so this never truncates a legitimate value. Mirrors the
+#: CLI's ``_MAX_OUTPUTS_LINE_CHARS`` posture (bound server text at the render
+#: boundary) rather than importing it — ``cli`` imports THIS module, so the
+#: dependency cannot run the other way.
+_MAX_SERVER_TEXT_CHARS = 200
+# Mirrors the control plane's bounded legacy-prefix query. A larger payload
+# cannot be a response from the supported resolver contract.
+_MAX_AMBIGUITY_CANDIDATES = 6
+
+
+def _output_record_from_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    """Validate and normalize one server output row through its contract.
+
+    ``OutputRecord.from_payload`` is deliberately used only at this response
+    deserialization boundary: it tolerates fields added by a newer server,
+    while direct ``OutputRecord(...)`` construction remains strict.  The
+    thin client's pre-existing mapping return shape is preserved for callers.
+    """
+    record = OutputRecord.from_payload(payload)
+    normalized = asdict(record)
+    if normalized.get("public_id") is None:
+        normalized.pop("public_id", None)
+    return normalized
+
+
+def _safe_server_text(value: Any) -> str:
+    """Make one server-derived value safe to interpolate into an error message.
+
+    **The choke point EVERY server-derived component of the digest-mismatch
+    raises below passes through — the digest AND the destination path.** That
+    "every component" framing is the whole point and was learned the hard way
+    twice on this PR: sanitizing one field while a sibling on the SAME LINE
+    stays raw neutralizes nothing, because the attacker simply moves the
+    payload one field over. Here that sibling was ``dest_path``, which is
+    built from a portable canonical server name for directory-form downloads
+    — and that helper rejects raw or normalization-created path separators,
+    vets filesystem shape, and removes terminal/Unicode-format controls from
+    the basename. This error boundary
+    still sanitizes every component independently as defense in depth and for
+    user-supplied explicit destination paths. Measured on the PR-B11 final round through
+    ``cli.main(["outputs", JOB, AID])``, with a wrong digest (which any
+    hostile server can force at will, needing no control of the digest
+    itself) and a name of ``report\\x1b[31mHACKED\\x1b[0m.bin``, the raw
+    interpolation emitted a live ESC to stderr inside the very message whose
+    purpose is to report a verification FAILURE.
+
+    Two hazards, handled in one place:
+
+    * Terminal control characters — the thin client's shared
+      ``sanitize_server_text`` helper, the
+      package's single choke point for server text bound for a terminal (the
+      same one the CLI's ``_safe_output_detail_text`` builds on), plus the
+      lone-surrogate round-trip that stripping does not cover (a ``\\udcff``
+      in a name reaches ``print()`` as ``UnicodeEncodeError: surrogates not
+      allowed`` — a stack trace where an error message belongs).
+      ``.lower().removeprefix("sha256:")`` on the digest is NOT a sanitizer;
+      measured, all of these survive it intact: ``\\x1b[31mHACKED\\x1b[0m``
+      (SGR — its final byte ``m`` is already lowercase),
+      ``\\x1b]8;;https://evil.example\\x07trusted.simulo.ai\\x1b]8;;\\x07``
+      (an OSC 8 hyperlink rendering as a trusted domain pointing elsewhere),
+      ``deadbeef\\rok — sha256 verified`` (a CR overwrite), and NUL.
+      Lowercasing happens to neuter some CSI sequences (``\\x1b[2K`` ->
+      ``\\x1b[2k``, not a valid final byte) but nothing else.
+    * UNBOUNDED LENGTH — neither field is length-checked anywhere on the way
+      here. Measured: a 50,000-char server name yielded a 50,351-char error
+      message. Bounded at :data:`_MAX_SERVER_TEXT_CHARS`.
+    """
+    text = sanitize_server_text(str(value))
+    text = text.encode("utf-8", "replace").decode("utf-8")
+    if len(text) > _MAX_SERVER_TEXT_CHARS:
+        text = text[: _MAX_SERVER_TEXT_CHARS - 1] + "…"
+    return text
+
+
+def _safe_server_repr(value: Any) -> str:
+    """``repr(value)`` for a server-derived value, bounded in width.
+
+    ``repr`` escapes terminal control bytes and lone surrogates, leaving width
+    as the remaining hazard. The helper remains part of the module's pinned
+    defensive contract even though ordinary identifier errors no longer echo
+    machine identifiers.
+    """
+    text = repr(value)
+    if len(text) > _MAX_SERVER_TEXT_CHARS:
+        text = text[: _MAX_SERVER_TEXT_CHARS - 1] + "…"
+    return text
+
+
+#: The wire name the `--from <job-ref>` seed rides under (contract-pinned pair;
+#: the CLI never sends the model-id sibling — the operator dropped
+#: `--from-model` from the human surface, so a user never types a raw UUID).
+_SEED_FROM_JOB_ID_FIELD, _SEED_FROM_MODEL_ID_FIELD = SEED_SOURCE_FIELDS
+
+
+class SubmitApiError(http.HttpError):
+    """Base error for the submit/recordings API."""
+
+
+class _AmbiguousJobRefError(SubmitApiError):
+    """Client-authored copy for the named legacy UUID-prefix exception."""
+
+    code = "job_ref_ambiguous"
+
+    def __init__(self, message: str, *, candidates: tuple[str, ...]) -> None:
+        super().__init__(message)
+        self.candidates = candidates
+
+
+def _validated_job_ref_ambiguity_candidates(exc: http.HttpHTTPError) -> Optional[tuple[str, ...]]:
+    """Canonical UUID candidates for the one named machine-ID exception.
+
+    The server owns the raw error envelope, so its ``code`` alone is not a
+    provenance boundary. Only a 409 with a non-empty, bounded list containing
+    2–6 distinct exact UUID strings is eligible for human display; malformed
+    or duplicate-bearing lists fail closed and ordinary UUID redaction remains
+    in force.
+    """
+    if exc.status != 409 or exc.code != "job_ref_ambiguous":
+        return None
+    raw_candidates = exc.candidates
+    if not isinstance(raw_candidates, list) or not raw_candidates or len(raw_candidates) > _MAX_AMBIGUITY_CANDIDATES:
+        return None
+    candidates: list[str] = []
+    for raw_candidate in raw_candidates:
+        if not isinstance(raw_candidate, str):
+            return None
+        try:
+            canonical = str(UUID(raw_candidate))
+        except (ValueError, AttributeError):
+            return None
+        if raw_candidate != canonical:
+            return None
+        if canonical in candidates:
+            return None
+        candidates.append(canonical)
+    return tuple(candidates) if len(candidates) >= 2 else None
+
+
+class AssetPinError(SubmitApiError):
+    """A submit-time asset failure — an unpinned ref under ``--frozen``, a ref that
+    could not be resolved, or a warning promoted to an error under
+    ``--strict-assets`` (see ``asset_pins.resolve_and_pin_assets``).
+
+    A ``SubmitApiError`` subclass so ``simulo run``'s existing handler renders it
+    as a clean one-liner (never a traceback) alongside every other submit error —
+    the failure happens in seconds, before the package is uploaded.
+    """
+
+
+class SeedNotHonoredError(SubmitApiError):
+    """A seed was requested but the created job record carries no seed provenance.
+
+    The loud-degradation guard for ``--from`` (explicit-run-intent plan): an
+    older control plane that predates the seed fields ignores them as unknown
+    keys and creates a FRESH job — which would silently train the wrong thing
+    and report success, a wrong-result hazard, not a cosmetic one. Unlike
+    ``simulo view``'s missing-route degradation (a warning-shaped fallback),
+    this is an ERROR: the job was already created without the seed by the
+    time the response reveals the problem, so the message names the created
+    Job ID when available (or gives a neutral lookup path) and explains how to
+    stop it. The client deliberately does NOT auto-cancel:
+    a 200 idempotent-replay response can attach to a pre-existing fresh job
+    the user started on purpose, and the wire response shape doesn't
+    distinguish 201-created from 200-replayed — cancelling would risk killing
+    a deliberate run, so the human makes that call.
+    """
+
+
+def _seed_error_from_http(exc: http.HttpHTTPError) -> Optional[SubmitApiError]:
+    """Map a seed-validation error from ``POST /v1/jobs`` to actionable copy.
+
+    Returns ``None`` for codes this function doesn't recognize (the caller
+    re-raises the original error unchanged — "surface the server's error"
+    rather than inventing text for unknown failures). The recognized codes
+    are the explicit-run-intent contract set (control plane
+    ``jobs/service.py::resolve_seed_source`` + ``jobs/job_ref.py``):
+
+    * ``job_ref_ambiguous`` (409) — the prefix matched several jobs; the
+      candidates are RENDERED (one per line), never silently picked from.
+    * ``job_not_found`` (404) — unknown/cross-org seed job: the same
+      friendly not-found copy ``simulo cancel``/``simulo view`` use, aimed
+      at the SEED reference rather than the submitted job.
+    * ``seed_artifact_invalid`` (422), ``seed_job_not_terminal`` /
+      ``seed_model_unavailable`` / ``seed_model_not_checkpoint`` /
+      ``seed_replay_conflict`` (409), ``seed_source_conflict`` (422) — the
+      server's message already says what is wrong; it is framed as CLI copy,
+      without echoing a caller-supplied legacy UUID.
+    """
+    if exc.code == "job_ref_ambiguous":
+        candidates = _validated_job_ref_ambiguity_candidates(exc)
+        if candidates is None:
+            return SubmitApiError(
+                "The --from selection is ambiguous, but the server did not provide valid UUID candidates. "
+                "List jobs with `simulo jobs` and use a complete Job ID."
+            )
+        listing = "".join(f"\n    {candidate}" for candidate in candidates)
+        return _AmbiguousJobRefError(
+            f"The --from selection is ambiguous — it matches more than one job:{listing}\n"
+            "Use a complete Job ID (see `simulo jobs`).",
+            candidates=candidates,
+        )
+    if exc.code == "job_not_found":
+        return SubmitApiError(
+            "The --from job doesn't exist, or you don't have access to it. List runs with `simulo jobs`."
+        )
+    if exc.code in (
+        "seed_artifact_invalid",
+        "seed_job_not_terminal",
+        "seed_model_unavailable",
+        "seed_model_not_checkpoint",
+        "seed_replay_conflict",
+        SEED_SOURCE_CONFLICT_CODE,
+    ):
+        return SubmitApiError(f"The --from selection could not be used: {exc.message}")
+    return None
+
+
+class SubmitApiClient:
+    """Client for package submit, job create, and recordings — the cloud path."""
+
+    def __init__(self, base_url: str, *, token: Optional[str] = None) -> None:
+        if not base_url.startswith(("http://", "https://")):
+            raise SubmitApiError(f"API base URL must be an http(s) URL, got {base_url!r}.")
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+
+    @property
+    def base_url(self) -> str:
+        return self._base_url
+
+    # ------------------------------------------------------------------
+    # Package submit + job create
+    # ------------------------------------------------------------------
+
+    def create_package(
+        self,
+        *,
+        package_id: str,
+        source_digest: str,
+        job_name: str,
+        archive_sha256: str,
+        archive_size: int,
+    ) -> dict[str, Any]:
+        """``POST /v1/packages`` — register a content-addressed package.
+
+        A 409 ``package_conflict`` is re-raised as actionable CLI copy:
+        the server already holds a package under this exact
+        ``package_id`` whose archive bytes differ. ``package_id`` is a pure
+        function of (source, job, args), so for a user who changed nothing
+        this almost always means a DIFFERENT ``simulo`` client version
+        packaged the same app differently (the packaging format moved) —
+        not a mistake in their code, and the raw wire code gives them
+        nothing to act on. The message states the one real remedy available
+        today (any source or args change mints a fresh submission identity)
+        and what the platform will do about the underlying limitation. The
+        server-side conflict check itself is correct and stays — one
+        package id must never silently refer to two different payloads.
+        """
+        try:
+            payload = self._post_json(
+                PACKAGES_ROUTE,
+                {
+                    "package_id": package_id,
+                    "source_digest": source_digest,
+                    "job_name": job_name,
+                    "archive_sha256": archive_sha256,
+                    "archive_size": archive_size,
+                },
+            )
+        except http.HttpHTTPError as exc:
+            if exc.status == 409 and exc.code == "package_conflict":
+                raise SubmitApiError(
+                    "This exact submission (same source, job, and args) is already registered "
+                    "on the platform with different package bytes — usually because a different "
+                    "simulo client version packaged it, not because of anything in your app. "
+                    "Nothing was uploaded and no job was created. To resubmit now, change "
+                    "anything about the app's source (even a comment) or its args so the "
+                    "submission gets a fresh identity; a platform update that accepts unchanged "
+                    "resubmissions automatically is planned."
+                ) from exc
+            raise
+        if not isinstance(payload, dict):
+            raise SubmitApiError(f"Malformed create_package response: {payload!r}")
+        return payload
+
+    def upload_archive(self, package_id: str, archive_bytes: bytes) -> None:
+        """``PUT /v1/packages/{id}/archive`` — upload the raw tar bytes.
+
+        Idempotent per the contract: re-uploading the same content-addressed
+        package id is a no-op.
+        """
+        path = PACKAGE_ARCHIVE_ROUTE_TEMPLATE.format(package_id=http.quote_path_segment(package_id))
+        http.request_bytes(
+            "PUT",
+            self._base_url + path,
+            token=self._token,
+            api_base_url=self._base_url,
+            body=archive_bytes,
+            content_type="application/octet-stream",
+            timeout=_UPLOAD_TIMEOUT_S,
+            unavailable_hint=_UNAVAILABLE_HINT,
+        )
+
+    def resolve_assets(self, refs: list[str], runtime: str) -> AssetsResolveResponse:
+        """``POST /v1/assets/resolve`` — resolve a batch of registry refs to exact
+        pins.
+
+        ``refs`` are the refs AS AUTHORED (short, possibly unpinned); the response
+        is keyed by those same strings so the caller correlates every result back
+        to its own ref. ``runtime`` is REQUIRED — the control plane computes
+        ``validated_on_runtime`` against it. Refuses a batch larger than
+        :data:`MAX_ASSETS_RESOLVE_BATCH` locally (a clear error rather than a
+        wire 422), and parses the two maps into the frozen wire dataclasses.
+        """
+        if len(refs) > MAX_ASSETS_RESOLVE_BATCH:
+            raise AssetPinError(
+                f"too many assets to resolve in one submit ({len(refs)}; the limit is "
+                f"{MAX_ASSETS_RESOLVE_BATCH}). Reduce the number of mounted assets."
+            )
+        payload = self._post_json(ASSETS_RESOLVE_ROUTE, {"refs": refs, "runtime": runtime})
+        if (
+            not isinstance(payload, dict)
+            or not isinstance(payload.get("resolved"), dict)
+            or not isinstance(payload.get("missing"), dict)
+        ):
+            raise SubmitApiError(f"Malformed assets/resolve response: {payload!r}")
+        try:
+            resolved = {
+                str(ref): ResolvedAssetPin(
+                    canonical_ref=entry["canonical_ref"],
+                    version=int(entry["version"]),
+                    digest=entry["digest"],
+                    kind=entry["kind"],
+                    deprecated=bool(entry["deprecated"]),
+                    validated_on_runtime=bool(entry["validated_on_runtime"]),
+                )
+                for ref, entry in payload["resolved"].items()
+            }
+            missing = {
+                str(ref): ResolveMiss(code=str(entry.get("code", "")), message=str(entry.get("message", "")))
+                for ref, entry in payload["missing"].items()
+            }
+        except (KeyError, TypeError, ValueError) as exc:
+            raise SubmitApiError(f"Malformed assets/resolve response: {payload!r} ({exc})") from exc
+        return AssetsResolveResponse(resolved=resolved, missing=missing)
+
+    def create_job(
+        self,
+        *,
+        package_id: str,
+        job_name: str,
+        args: dict[str, Any],
+        viewstream: bool = False,
+        seed_from_job_id: Optional[str] = None,
+        assets: Optional[list[dict[str, str]]] = None,
+        app_name: Optional[str] = None,
+        system: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """``POST /v1/jobs`` — create the job that executes *package_id*.
+
+        ``app_name`` (live-view wave) is the submitting ``App``'s
+        declared name — display context the platform's live viewer shows next
+        to ``job_name``. An OPTIONAL top-level sibling of ``args`` following
+        the exact same package-hash-invariance rule as ``viewstream`` /
+        ``seed_from_job_id`` / ``assets`` below (``args``' canonical JSON
+        feeds the content-addressed ``package_id``; a display property must
+        never perturb package identity). Sent only when non-empty; a control
+        plane that predates the field ignores the extra key, and a job
+        without one simply renders by ``job_name`` alone — best-effort
+        metadata, never load-bearing.
+
+        ``viewstream`` is an OPTIONAL top-level body field — a sibling of
+        ``args``, never nested inside it (``args`` feeds the ``package_id``
+        sha256; a streaming flag must never change a package's content-
+        addressed identity). Sent only when ``True``: a control plane that
+        predates the live-visualization plan's ``JobSubmitRequest`` field
+        simply ignores an absent key, same as it ignores any unknown field.
+
+        ``system`` (system-tier-catalog-1362, M50-persistence) is the
+        already-validated tier wire value (e.g. ``"tier1"``) from the job's
+        own manifest ``resources["system"]`` — an OPTIONAL top-level sibling
+        of ``args`` for the same package-hash-invariance reason as
+        ``viewstream`` above. Sent only when truthy: a control plane that
+        predates this field ignores the extra key, and the run proceeds
+        exactly as it did before ``system=`` existed.
+
+        ``seed_from_job_id`` (``simulo run --from``, explicit-run-intent
+        plan) follows the exact same sibling-of-``args`` rule for the exact
+        same package-hash reason — a seed is a property of THIS submission,
+        never of the package. The value is passed through VERBATIM
+        (``<job-ref>[:best|:latest]``, ref = complete Job ID, full UUID, or an
+        unambiguous UUID prefix): the control plane owns resolution, org-scoped, in one
+        round trip — the client never resolves a model id itself. Two
+        seed-specific behaviors on top of the plain call:
+
+        * seed-validation failures are re-raised with actionable CLI copy
+          (:func:`_seed_error_from_http`; unrecognized codes propagate
+          unchanged);
+        * a 2xx response whose record carries NO ``seed`` provenance means
+          an older control plane silently ignored the request — that job
+          would train FRESH while claiming success, so it raises
+          :class:`SeedNotHonoredError` instead of returning (see that
+          class's docstring for why it never auto-cancels).
+
+        ``assets`` (submit-time pin resolution) is the resolved
+        ``[{ref, digest}]`` list — a top-level sibling of ``args``, on the
+        :data:`~simulo.interfaces.platform.asset_catalog.ASSETS_FIELD` key,
+        following the EXACT same package-hash-invariance rule as ``viewstream`` /
+        ``seed_from_job_id``: an asset pin is a property of THIS submission, never
+        of the package, so it must never enter ``args`` (whose canonical JSON
+        feeds ``package_id``). Sent only when non-empty; the control plane
+        re-verifies every pin in the submit transaction and writes ``job_assets``.
+
+        Replay conflicts: the control plane's submit is idempotent on the
+        package's in-flight job, and refuses (409) a replay whose intent
+        differs from that job's rather than silently answering with it.
+        Both replay-conflict codes are mapped to :class:`SubmitApiError`
+        here, the ``_seed_error_from_http`` convention — unmapped they
+        would render as a raw wire dump
+        (``error: viewstream_replay_conflict: ...``) rather than copy
+        framed around what this submit asked for:
+
+        * ``viewstream_replay_conflict`` — framed with whether THIS submit
+          asked for ``--viewstream``, since either direction can conflict;
+        * ``seed_replay_conflict`` with NO ``--from`` on this submit — the
+          fresh-resubmit-of-a-seeded-job direction, which the ``--from``-
+          gated :func:`_seed_error_from_http` branch below cannot reach
+          (when ``--from`` WAS passed, that branch keeps handling it).
+          There is no seed REF to frame with, but the flag's absence is a
+          user-visible fact, so the copy is framed on it (``without
+          --from``) — the same treatment as the viewstream direction.
+
+        Returns the :class:`~simulo.interfaces.platform.runs.JobRecord`
+        fields. The HTTP status is the created-vs-replayed signal — 201 on
+        first submit, 200 on an idempotent replay of an in-flight job — and
+        it is the ONLY signal: the body is exactly the JobRecord fields on
+        both paths (kept free of extras so ``JobRecord(**payload)``
+        reconstruction stays valid), and this client treats both statuses
+        alike.
+        """
+        body: dict[str, Any] = {"package_id": package_id, "job_name": job_name, "args": args}
+        if app_name:
+            body["app_name"] = app_name
+        if viewstream:
+            body["viewstream"] = True
+        if system:
+            body["system"] = system
+        if seed_from_job_id is not None:
+            body[_SEED_FROM_JOB_ID_FIELD] = seed_from_job_id
+        if assets:
+            body[ASSETS_FIELD] = assets
+        try:
+            payload = self._post_json(JOBS_ROUTE, body)
+        except http.HttpHTTPError as exc:
+            # NOTE (#830/#838 fix round, measured): an unmapped HttpHTTPError
+            # never tracebacks out of `simulo run` — cli.py's
+            # `except (JobsApiError, SubmitApiError)` catches it, because
+            # JobsApiError is an ALIAS of HttpError (jobs_api.py). These
+            # mappings exist to upgrade the resulting `error: <wire-code>:
+            # <message>` dump to framed CLI copy, not to prevent a crash.
+            if exc.code == "viewstream_replay_conflict":
+                requested = "with --viewstream" if viewstream else "without --viewstream"
+                raise SubmitApiError(f"Cannot re-submit {requested}: {exc.message}") from exc
+            if seed_from_job_id is not None:
+                friendly = _seed_error_from_http(exc)
+                if friendly is not None:
+                    raise friendly from exc
+            elif exc.code == "seed_replay_conflict":
+                # NOTE: this elif pairs with `if seed_from_job_id is not
+                # None:` above — NOT with the nested `if friendly is not
+                # None:` that ends that block. It is the no---from direction
+                # of the same server guard (a FRESH re-submit of a package
+                # whose in-flight job was seeded), reachable only when no
+                # seed was requested on THIS submit. _seed_error_from_http
+                # frames its copy around the user's --from value, which does
+                # not exist here — so frame on the flag's ABSENCE instead;
+                # the server's message names the in-flight job and remedy.
+                raise SubmitApiError(f"Cannot re-submit without --from: {exc.message}") from exc
+            raise
+        if not isinstance(payload, dict) or not payload.get("job_id"):
+            raise SubmitApiError("Malformed create_job response: no machine job identity was returned.")
+        if seed_from_job_id is not None:
+            seed = payload.get("seed")
+            if not isinstance(seed, dict) or not seed.get("model_id"):
+                try:
+                    raw_public_id = payload.get("public_id")
+                    if not isinstance(raw_public_id, str):
+                        raise TypeError("public_id is not a string")
+                    public_id = str(parse_job_public_id(raw_public_id))
+                except (TypeError, ValueError):
+                    public_id = None
+                created_job = f"Job {public_id}" if public_id is not None else "The job"
+                cancel_hint = (
+                    f"`simulo cancel {public_id}`" if public_id is not None else "`simulo jobs`, then `simulo cancel`"
+                )
+                raise SeedNotHonoredError(
+                    "The --from selection was NOT honored: this control plane does not "
+                    "support continuing a run from a checkpoint (it ignored the request), so the "
+                    f"job it created would train FRESH — the wrong result, reported as success.\n"
+                    f"{created_job} was created WITHOUT the checkpoint. Stop it with {cancel_hint} "
+                    "if you did not intend a fresh run, and re-run without --from "
+                    "(or against a control plane that supports it)."
+                )
+        return payload
+
+    # ------------------------------------------------------------------
+    # Recordings
+    # ------------------------------------------------------------------
+
+    def list_recordings(self, job_id: str, *, scope: str = JOB_SCOPE_MINE) -> list[dict[str, Any]]:
+        """All :class:`RecordingRecord` entries for *job_id*, across every page."""
+        path = JOB_RECORDINGS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        items: list[dict[str, Any]] = []
+        page = 1
+        while page <= _LIST_MAX_PAGES:
+            payload = self._get(path, self._page_query(_LIST_PAGE_LIMIT, page, scope))
+            page_items = payload.get("items") if isinstance(payload, dict) else None
+            if not isinstance(page_items, list):
+                raise SubmitApiError("Malformed recordings list response (no 'items' array).")
+            items.extend(item for item in page_items if isinstance(item, dict))
+            if len(page_items) < _LIST_PAGE_LIMIT:
+                return items
+            page += 1
+        return items
+
+    def list_recordings_page(
+        self, job_id: str, *, limit: int, page: int = 1, scope: str = JOB_SCOPE_MINE
+    ) -> tuple[list[dict[str, Any]], int]:
+        """One page (up to *limit* rows) of *job_id*'s recordings, plus the
+        total recording count for the job.
+
+        The single-request counterpart to :meth:`list_recordings` (which
+        loops every page — still used for ``simulo recordings --all`` and
+        NAME resolution, both of which need the complete set). Backs the
+        CLI's default recent-N view / ``--limit N`` in the bare (no NAME, no
+        ``--all``) branch only; *page* defaults to the first, and the CLI's
+        ``_fetch_list_view`` walks it forward for a view wider than one
+        server page.
+        """
+        path = JOB_RECORDINGS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        payload = self._get(path, self._page_query(limit, page, scope))
+        items = payload.get("items") if isinstance(payload, dict) else None
+        total = payload.get("total") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not isinstance(total, int):
+            raise SubmitApiError("Malformed recordings list response (no 'items'/'total').")
+        return [item for item in items if isinstance(item, dict)], total
+
+    def get_recording_download_url(
+        self, job_id: str, recording_id: str, *, scope: str = JOB_SCOPE_MINE
+    ) -> dict[str, Any]:
+        """``GET .../recordings/{id}/download`` — resolve a short-lived download link."""
+        path = JOB_RECORDING_DOWNLOAD_ROUTE_TEMPLATE.format(
+            job_id=http.quote_path_segment(job_id), recording_id=http.quote_path_segment(recording_id)
+        )
+        payload = self._get(path, {"scope": scope})
+        if not isinstance(payload, dict) or not payload.get("url"):
+            raise SubmitApiError("Malformed recording download-link response.")
+        return payload
+
+    def download_recording(
+        self,
+        job_id: str,
+        recording_id: str,
+        dest_path: Path,
+        *,
+        scope: str = JOB_SCOPE_MINE,
+        destination_observation: Optional[DestinationObservation] = None,
+        replace_existing: bool = True,
+    ) -> Path:
+        """Resolve and download one recording's MCAP bytes to *dest_path*.
+
+        The bearer is attached to the download URL only when its host equals
+        the API host (see ``http.may_attach_bearer`` — the foreign-host rule
+        for presigned S3 URLs).
+
+        Any failure after temp creation may leave a mode-0600 ``.part`` file
+        because no backend can prove a pathname still identifies this
+        invocation's temp entry before deleting it. Ordinary write failures
+        name the retained entry; interrupts and process exits propagate
+        unchanged. The client never automatically deletes a prior ``.part``
+        file.
+
+        SECURITY (destination-directory race hardening): the destination
+        directory is SNAPSHOTTED before any network I/O begins
+        (``observe_destination`` — read-only, so a failed fetch still
+        leaves zero filesystem footprint, exactly as before), and the
+        eventual atomic write (same-directory temp file + fsync + rename)
+        is bound to that snapshot (``open_destination_directory``), never
+        to a fresh, unqualified lookup of the path string taken after the
+        fetch. That binding — not mere adjacency of two statements — is
+        what closes the race across the fetch's entire duration, including
+        every ancestor a nested ``-o a/b/c`` would create. The observation
+        may hold an open file descriptor (POSIX) for the entire fetch, so
+        it is released in a ``finally`` on every path — a digest mismatch
+        or fetch failure included, not only the success path. See
+        ``_secure_downloads`` for exactly what this closes and the
+        documented POSIX/Windows difference.
+        """
+        destination = destination_observation or observe_destination(dest_path.parent)
+        if not destination.matches_directory(dest_path.parent):
+            destination.close()
+            raise DestinationDirectoryError(
+                f"Refusing to use an observation for {destination.expanded} to write {dest_path}."
+            )
+        try:
+            link = self.get_recording_download_url(job_id, recording_id, scope=scope)
+            download_url = str(link["url"])
+            data = http.request_bytes(
+                "GET",
+                download_url,
+                token=self._token,
+                api_base_url=self._base_url,
+                timeout=_DOWNLOAD_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+            )
+            destination_dir = open_destination_directory(destination)
+            try:
+                destination_dir.write_atomic(dest_path.name, data, replace_existing=replace_existing)
+            finally:
+                destination_dir.close()
+        finally:
+            destination.close()
+        return dest_path
+
+    # ------------------------------------------------------------------
+    # Models (trained checkpoints) — the recordings surface's analogue
+    # ------------------------------------------------------------------
+
+    def list_models(self, job_id: str, *, scope: str = JOB_SCOPE_MINE) -> list[dict[str, Any]]:
+        """All :class:`ModelRecord` entries for *job_id*, across every page."""
+        path = JOB_MODELS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        items: list[dict[str, Any]] = []
+        page = 1
+        while page <= _LIST_MAX_PAGES:
+            payload = self._get(path, self._page_query(_LIST_PAGE_LIMIT, page, scope))
+            page_items = payload.get("items") if isinstance(payload, dict) else None
+            if not isinstance(page_items, list):
+                raise SubmitApiError("Malformed models list response (no 'items' array).")
+            items.extend(item for item in page_items if isinstance(item, dict))
+            if len(page_items) < _LIST_PAGE_LIMIT:
+                return items
+            page += 1
+        return items
+
+    def list_models_page(
+        self, job_id: str, *, limit: int, page: int = 1, scope: str = JOB_SCOPE_MINE
+    ) -> tuple[list[dict[str, Any]], int]:
+        """One page (up to *limit* rows) of *job_id*'s models, plus the total
+        model count for the job.
+
+        The single-request counterpart to :meth:`list_models` (which loops
+        every page — still used for ``simulo models --all`` and NAME/id
+        resolution, both of which need the complete set). Backs the CLI's
+        default recent-N view / ``--limit N`` in the bare (no NAME, no
+        ``--all``) branch only; *page* defaults to the first, and the CLI's
+        ``_fetch_list_view`` walks it forward for a view wider than one
+        server page.
+        """
+        path = JOB_MODELS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        payload = self._get(path, self._page_query(limit, page, scope))
+        items = payload.get("items") if isinstance(payload, dict) else None
+        total = payload.get("total") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not isinstance(total, int):
+            raise SubmitApiError("Malformed models list response (no 'items'/'total').")
+        return [item for item in items if isinstance(item, dict)], total
+
+    def get_model(self, job_id: str, model_id: str, *, scope: str = JOB_SCOPE_MINE) -> dict[str, Any]:
+        """``GET .../models/{id}`` — one model's metadata record."""
+        path = JOB_MODEL_ROUTE_TEMPLATE.format(
+            job_id=http.quote_path_segment(job_id), model_id=http.quote_path_segment(model_id)
+        )
+        payload = self._get(path, {"scope": scope})
+        if not isinstance(payload, dict) or not payload.get("model_id"):
+            raise SubmitApiError("Malformed model record response.")
+        return payload
+
+    def get_model_download_url(self, job_id: str, model_id: str, *, scope: str = JOB_SCOPE_MINE) -> dict[str, Any]:
+        """``GET .../models/{id}/download`` — resolve a short-lived download link."""
+        path = JOB_MODEL_DOWNLOAD_ROUTE_TEMPLATE.format(
+            job_id=http.quote_path_segment(job_id), model_id=http.quote_path_segment(model_id)
+        )
+        payload = self._get(path, {"scope": scope})
+        if not isinstance(payload, dict) or not payload.get("url"):
+            raise SubmitApiError("Malformed model download-link response.")
+        return payload
+
+    def download_model(
+        self,
+        job_id: str,
+        model_id: str,
+        dest_path: Path,
+        *,
+        expected_sha256: Optional[str] = None,
+        scope: str = JOB_SCOPE_MINE,
+        destination_observation: Optional[DestinationObservation] = None,
+        replace_existing: bool = True,
+    ) -> Path:
+        """Resolve and download one model's checkpoint bytes to *dest_path*.
+
+        When *expected_sha256* is given (the record's ``digest_sha256`` —
+        always present for models), the downloaded bytes are verified
+        against it BEFORE anything is written to *dest_path*: a mismatch
+        raises :class:`SubmitApiError` and leaves no partial/corrupt file
+        behind. The bearer is attached to the download URL only when its
+        host equals the API host (``http.may_attach_bearer`` — the
+        foreign-host rule for presigned S3 URLs).
+
+        The write itself is ATOMIC (a deliberate security hardening): the
+        bytes land in a temp file created in *dest_path*'s OWN directory
+        (guaranteeing the same filesystem, so the final rename is atomic on
+        POSIX and Windows alike) and are only ``os.replace``'d into
+        *dest_path* after every byte is written and fsync'd. A killed
+        process, a full disk mid-write, or two concurrent ``simulo models``
+        invocations targeting the same path can therefore never leave a
+        truncated/torn file at *dest_path* — a reader always sees either the
+        prior complete file (if any) or the new complete one, never a
+        partial write in between.
+
+        Any failure after temp creation may leave a mode-0600 ``.part`` file
+        because no backend can prove a pathname still identifies this
+        invocation's temp entry before deleting it. Ordinary write failures
+        name the retained entry; interrupts and process exits propagate
+        unchanged. The client never automatically deletes a prior ``.part``
+        file.
+
+        SECURITY (destination-directory race hardening): the destination
+        directory is SNAPSHOTTED before any network I/O begins — before
+        even the download URL is resolved (``observe_destination`` —
+        read-only, so a digest mismatch or failed fetch still leaves zero
+        filesystem footprint, exactly as before, not even the destination
+        directory created) — and the eventual atomic write is bound to
+        that snapshot (``open_destination_directory``), never to a fresh,
+        unqualified lookup of the path string taken after the fetch. That
+        binding — not mere adjacency of two statements — is what closes
+        the race across the fetch's entire duration, including every
+        ancestor a nested ``-o a/b/c`` would create. The observation may
+        hold an open file descriptor (POSIX) for the entire fetch, so it
+        is released in a ``finally`` on every path — a digest mismatch or
+        fetch failure included, not only the success path. See
+        ``_secure_downloads`` for exactly what this closes and the
+        documented POSIX/Windows difference.
+
+        ``dest_path`` is the caller's own destination, but its BASENAME is
+        routinely server-derived. Directory-form CLI callers build it from
+        ``_portable_directory_download_filename(record["name"])``; that
+        helper rejects raw or normalization-created path separators, chooses
+        a portable canonical basename, rejects NUL and Windows-unsafe forms,
+        and removes Unicode control/format characters. So *dest_path* remains
+        untrusted display text wherever
+        this method renders it, and the mismatch raise above routes it
+        through :func:`_safe_server_text` as defense in depth. Callers remain
+        responsible for the filesystem half before constructing it.
+        """
+        destination = destination_observation or observe_destination(dest_path.parent)
+        if not destination.matches_directory(dest_path.parent):
+            destination.close()
+            raise DestinationDirectoryError(
+                f"Refusing to use an observation for {destination.expanded} to write {dest_path}."
+            )
+        try:
+            link = self.get_model_download_url(job_id, model_id, scope=scope)
+            download_url = str(link["url"])
+            data = http.request_bytes(
+                "GET",
+                download_url,
+                token=self._token,
+                api_base_url=self._base_url,
+                timeout=_DOWNLOAD_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+            )
+            if expected_sha256:
+                expected = expected_sha256.lower().removeprefix("sha256:")
+                actual = hashlib.sha256(data).hexdigest()
+                if actual != expected:
+                    raise SubmitApiError(
+                        "Model download digest mismatch for the selected Model: "
+                        f"expected sha256:{_safe_server_text(expected)}, "
+                        f"got sha256:{actual} ({len(data)} bytes). "
+                        f"Nothing was written to {_safe_server_text(dest_path)}."
+                    )
+            destination_dir = open_destination_directory(destination)
+            try:
+                destination_dir.write_atomic(dest_path.name, data, replace_existing=replace_existing)
+            finally:
+                destination_dir.close()
+        finally:
+            destination.close()
+        return dest_path
+
+    # ------------------------------------------------------------------
+    # Outputs (Plan B, #531, PR-B10) — the read-time UNION over recordings,
+    # models, and general job_artifacts. A projection, not a home: the same
+    # bytes stay downloadable from this route AND the output's original
+    # /recordings or /models surface (contract docstring on
+    # JOB_OUTPUT_DOWNLOAD_ROUTE_TEMPLATE) — this client method set exists
+    # so `simulo outputs` has one listing/download surface spanning all
+    # three, including the general kind (report/video/dataset/file/
+    # anomaly_capture) that has no other CLI download path.
+    # ------------------------------------------------------------------
+
+    def list_outputs(self, job_id: str, *, scope: str = JOB_SCOPE_MINE) -> list[dict[str, Any]]:
+        """All :class:`OutputRecord` entries for *job_id*, across every page.
+
+        ``scope`` (contract ``runs.JOB_SCOPE_MINE``/``JOB_SCOPE_ORG``) is sent
+        explicitly on every request. ``simulo export`` passes
+        ``scope="org"``: the export job that owns a policy bundle is
+        SYSTEM-submitted (``submitted_by_user_id=None``), so the server's
+        ``scope=mine`` default answers the non-enumerating 404 for it. The
+        same applies to :meth:`download_output`, which resolves access from
+        the same job.
+        """
+        path = JOB_OUTPUTS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        items: list[dict[str, Any]] = []
+        page = 1
+        while page <= _LIST_MAX_PAGES:
+            payload = self._get(path, self._page_query(_LIST_PAGE_LIMIT, page, scope))
+            page_items = payload.get("items") if isinstance(payload, dict) else None
+            if not isinstance(page_items, list):
+                raise SubmitApiError("Malformed outputs list response (no 'items' array).")
+            try:
+                items.extend(_output_record_from_payload(item) for item in page_items)
+            except (TypeError, ValueError) as exc:
+                raise SubmitApiError(f"Malformed outputs list response: {_safe_server_text(exc)}") from exc
+            if len(page_items) < _LIST_PAGE_LIMIT:
+                return items
+            page += 1
+        return items
+
+    def list_outputs_page(
+        self, job_id: str, *, limit: int, page: int = 1, scope: str = JOB_SCOPE_MINE
+    ) -> tuple[list[dict[str, Any]], int]:
+        """One page (up to *limit* rows) of *job_id*'s outputs, plus the
+        total output count for the job.
+
+        The single-request counterpart to :meth:`list_outputs` (which loops
+        every page — used for ``simulo outputs --all`` and NAME/id
+        resolution, both of which need the complete set). Backs the CLI's
+        default recent-N view / ``--limit N`` in the bare (no NAME, no
+        ``--all``) branch only; *page* defaults to the first, and the CLI's
+        ``_fetch_list_view`` walks it forward for a view wider than one
+        server page.
+        """
+        path = JOB_OUTPUTS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+        payload = self._get(path, self._page_query(limit, page, scope))
+        items = payload.get("items") if isinstance(payload, dict) else None
+        total = payload.get("total") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not isinstance(total, int):
+            raise SubmitApiError("Malformed outputs list response (no 'items'/'total').")
+        try:
+            records = [_output_record_from_payload(item) for item in items]
+        except (TypeError, ValueError) as exc:
+            raise SubmitApiError(f"Malformed outputs list response: {_safe_server_text(exc)}") from exc
+        return records, total
+
+    def get_output_download_url(self, job_id: str, output_id: str, *, scope: str = JOB_SCOPE_MINE) -> dict[str, Any]:
+        """``GET .../outputs/{id}/download`` — resolve a short-lived download link.
+
+        ``scope``: see :meth:`list_outputs`. The download route resolves
+        access from the OWNING job exactly as the listing does, so a scope that
+        is right for the listing and absent here is a 404 halfway through a
+        download — they must move together.
+        """
+        path = JOB_OUTPUT_DOWNLOAD_ROUTE_TEMPLATE.format(
+            job_id=http.quote_path_segment(job_id), output_id=http.quote_path_segment(output_id)
+        )
+        payload = self._get(path, {"scope": scope})
+        if not isinstance(payload, dict) or not payload.get("url"):
+            raise SubmitApiError("Malformed output download-link response.")
+        return payload
+
+    def download_output(
+        self,
+        job_id: str,
+        output_id: str,
+        dest_path: Path,
+        *,
+        expected_sha256: Optional[str] = None,
+        scope: str = JOB_SCOPE_MINE,
+        destination_observation: Optional[DestinationObservation] = None,
+        replace_existing: bool = True,
+    ) -> Path:
+        """Resolve and download one output's bytes to *dest_path*.
+
+        Mirrors :meth:`download_model`'s atomic-write pattern (a same-directory
+        temp file + ``os.fsync`` + ``os.replace``, so a killed process or a
+        full disk can never leave a truncated file at *dest_path*) rather than
+        the simpler ``download_recording`` write — a general output can be
+        as large as a video or dataset, and the same digest-verification path
+        applies when the caller has one.
+
+        Any failure after temp creation may leave a mode-0600 ``.part`` file
+        because no backend can prove a pathname still identifies this
+        invocation's temp entry before deleting it. Ordinary write failures
+        name the retained entry; interrupts and process exits propagate
+        unchanged. The client never automatically deletes a prior ``.part``
+        file.
+
+        *expected_sha256* is the record's ``digest_sha256``, verified BEFORE
+        anything is written when given. Pass ``None`` — not ``""`` — when the
+        record's digest is empty: :class:`~simulo.interfaces.platform.
+        outputs.OutputRecord` documents ``""`` as "no digest stored" for a
+        row whose home table records none (the recordings convention), which
+        means skip-verify, never a mismatch.
+
+        SECURITY (destination-directory race hardening): the destination
+        directory is SNAPSHOTTED before any network I/O begins — before
+        even the download URL is resolved (``observe_destination`` —
+        read-only, so a digest mismatch or failed fetch still leaves zero
+        filesystem footprint, exactly as before, not even the destination
+        directory created) — and the eventual atomic write is bound to
+        that snapshot (``open_destination_directory``), never to a fresh,
+        unqualified lookup of the path string taken after the fetch. That
+        binding — not mere adjacency of two statements — is what closes
+        the race across the fetch's entire duration, including every
+        ancestor a nested ``-o a/b/c`` would create. The observation may
+        hold an open file descriptor (POSIX) for the entire fetch, so it
+        is released in a ``finally`` on every path — a digest mismatch or
+        fetch failure included, not only the success path. See
+        ``_secure_downloads`` for exactly what this closes and the
+        documented POSIX/Windows difference.
+
+        ``scope``: see :meth:`list_outputs`.
+        """
+        destination = destination_observation or observe_destination(dest_path.parent)
+        if not destination.matches_directory(dest_path.parent):
+            destination.close()
+            raise DestinationDirectoryError(
+                f"Refusing to use an observation for {destination.expanded} to write {dest_path}."
+            )
+        try:
+            link = self.get_output_download_url(job_id, output_id, scope=scope)
+            download_url = str(link["url"])
+            data = http.request_bytes(
+                "GET",
+                download_url,
+                token=self._token,
+                api_base_url=self._base_url,
+                timeout=_DOWNLOAD_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+            )
+            if expected_sha256:
+                expected = expected_sha256.lower().removeprefix("sha256:")
+                actual = hashlib.sha256(data).hexdigest()
+                if actual != expected:
+                    raise SubmitApiError(
+                        "Output download digest mismatch for the selected Output: "
+                        f"expected sha256:{_safe_server_text(expected)}, "
+                        f"got sha256:{actual} ({len(data)} bytes). "
+                        f"Nothing was written to {_safe_server_text(dest_path)}."
+                    )
+            destination_dir = open_destination_directory(destination)
+            try:
+                destination_dir.write_atomic(dest_path.name, data, replace_existing=replace_existing)
+            finally:
+                destination_dir.close()
+        finally:
+            destination.close()
+        return dest_path
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _page_query(limit: int, page: int, scope: str) -> dict[str, str]:
+        """Build one explicitly scoped pagination query."""
+        return {"limit": str(limit), "page": str(page), "scope": scope}
+
+    def _get(self, path: str, query: Optional[dict[str, str]] = None) -> Any:
+        url = self._base_url + path
+        if query:
+            url += "?" + urllib.parse.urlencode(query)
+        return http.request_json(
+            "GET",
+            url,
+            token=self._token,
+            api_base_url=self._base_url,
+            timeout=_REQUEST_TIMEOUT_S,
+            unavailable_hint=_UNAVAILABLE_HINT,
+        )
+
+    def _post_json(self, path: str, body: dict[str, Any]) -> Any:
+        return http.request_json(
+            "POST",
+            self._base_url + path,
+            token=self._token,
+            api_base_url=self._base_url,
+            json_body=body,
+            timeout=_REQUEST_TIMEOUT_S,
+            unavailable_hint=_UNAVAILABLE_HINT,
+        )