simulo 0.26.0__py3-none-any.whl

simulo/_client/export_api.py

@@ -0,0 +1,212 @@
+"""HTTP client for the policy-export surface (stdlib only, torch-free).
+
+Implements the client half of ``POST``/``GET``
+:data:`~simulo.interfaces.platform.policy_bundle.JOB_EXPORTS_ROUTE_TEMPLATE`
+— request a portable policy export of one of a training job's checkpoints, and
+read back the export's status and the platform's own conversion-equivalence
+verdict. Request plumbing (structured errors, the https-when-token guard, the
+foreign-host bearer rule) is shared with ``jobs_api.py``/``submit_api.py`` via
+``http.py``.
+
+**The route and the vocabulary are imported, never retyped.** The path string,
+the export format enum, the verdict, and the four named failure reasons all
+live in ``simulo.interfaces.platform.policy_bundle``; this module holds no
+copy of any of them, so a contract change cannot leave a stale spelling here.
+
+**Scope.** Both verbs resolve access from the SOURCE job and carry one explicit
+``?scope=`` through create/list/poll. Direct SDK calls default to ``org``
+because naming a source job is explicit intent; id-less CLI flows resolve the
+latest caller-owned job first and then pass ``mine`` through these methods.
+
+That default would restrict exporting to jobs the caller personally submitted,
+which is wrong for the case the feature exists to serve: one engineer trains a
+policy and another puts it on a robot. Organization members share organization
+jobs and outputs under the current authorization model, so a member who can
+already read a teammate's job and download its checkpoints (``simulo models``)
+has no coherent reason to be refused its ONNX bundle. ``simulo cancel`` set the
+precedent — it resolves org-wide because naming a job explicitly is explicit
+intent, and the equal-standing model deliberately extends to it. It is hard to
+argue a read needs a stricter default than a destructive write.
+
+Sibling read methods (``simulo models``, ``simulo outputs``, and
+``simulo recordings``) expose the same explicit scope plumbing: named jobs use
+organization scope, while an id-less latest selection keeps caller scope.
+
+This widens nothing across an organization boundary: ``scope=org`` is still
+org-bounded, and the server re-checks membership on every request. Nor does it
+change the zero-argument path, which resolves "the latest job" as the caller's
+own before this client is reached.
+
+The EXPORT job the response names is system-submitted
+(``submitted_by_user_id=None``) and is therefore invisible under the default
+too: every read of the export job itself (its outputs, its download link)
+must pass ``scope=org``. The output methods on the submit client take an
+explicit ``scope`` for exactly this reason.
+"""
+
+from __future__ import annotations
+
+import urllib.parse
+from typing import Any, Optional
+
+from simulo._client import http
+from simulo.interfaces.platform.policy_bundle import JOB_EXPORTS_ROUTE_TEMPLATE, PolicyExportFormat
+from simulo.interfaces.platform.runs import JOB_SCOPE_ORG
+
+_REQUEST_TIMEOUT_S = 10.0  # every outbound call has an explicit timeout (NFR)
+
+#: Page size used when scanning a source job's exports for one export id.
+_LIST_PAGE_LIMIT = 100
+
+#: Pages :meth:`ExportApiClient.find_export` will scan before giving up. The
+#: listing is newest-first and an export is looked up right after it was
+#: created, so page 1 holds it unless 100+ later exports of the SAME source job
+#: were created in between; a handful of pages is generous headroom over that
+#: and keeps a poll loop's per-iteration cost bounded (a poll that could issue
+#: 1000 requests per tick is a self-inflicted rate limit).
+_LIST_MAX_PAGES = 5
+
+_UNAVAILABLE_HINT = "Check SIMULO_API_URL / SIMULO_ENV, and that you are logged in (`simulo login`)."
+
+#: The default portable format — the contract's enum, not a literal.
+DEFAULT_EXPORT_FORMAT = PolicyExportFormat.ONNX.value
+
+#: Scope both export verbs send when resolving the SOURCE job. Organization
+#: members share organization jobs and outputs, so a member who can already
+#: read a teammate's training run and download its checkpoints must not be
+#: refused its ONNX bundle. Org-bounded, and the server re-checks membership
+#: per request; the zero-argument path still resolves "latest" as the caller's
+#: own job before reaching this client.
+SOURCE_JOB_SCOPE = JOB_SCOPE_ORG
+
+
+class ExportApiError(http.HttpError):
+    """A malformed or unusable response from the export surface.
+
+    A ``RuntimeError`` subclass sharing ``http.HttpError``'s base with
+    ``JobsApiError``/``SubmitApiError``, so the CLI's shared observe-command
+    handler renders it as a clean one-liner rather than a traceback. Structured
+    non-2xx responses stay ``http.HttpHTTPError`` (also an ``HttpError``) so the
+    CLI can switch on the server's own error ``code``.
+    """
+
+
+class ExportApiClient:
+    """Client for ``POST``/``GET /v1/jobs/{job_id}/exports``."""
+
+    def __init__(self, base_url: str, *, token: Optional[str] = None) -> None:
+        if not base_url.startswith(("http://", "https://")):
+            raise ExportApiError(f"API base URL must be an http(s) URL, got {base_url!r}.")
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+
+    @property
+    def base_url(self) -> str:
+        return self._base_url
+
+    def create_export(
+        self,
+        job_id: str,
+        *,
+        export_format: str = DEFAULT_EXPORT_FORMAT,
+        model_id: Optional[str] = None,
+        kind: Optional[str] = None,
+        scope: str = SOURCE_JOB_SCOPE,
+    ) -> dict[str, Any]:
+        """``POST .../exports`` — request an export; return the export record.
+
+        ``model_id`` and ``kind`` are OVERRIDES and are sent only when the user
+        typed one: the platform's default (this job's newest ``best``
+        checkpoint) is an answer Simulo already knows, so the
+        zero-argument path posts nothing but the format.
+
+        The status code is the created-vs-replayed signal (201 new, 200 an
+        in-flight export of the same source job/checkpoint/format returned
+        as-is) and both bodies are the same shape, so this method treats them
+        alike — pressing the button twice attaches to the export already on its
+        way rather than paying for a second one.
+        """
+        body: dict[str, Any] = {"format": export_format}
+        if model_id is not None:
+            body["model_id"] = model_id
+        if kind is not None:
+            body["kind"] = kind
+        payload = http.request_json(
+            "POST",
+            self._base_url + self._path(job_id) + "?" + urllib.parse.urlencode({"scope": scope}),
+            token=self._token,
+            api_base_url=self._base_url,
+            json_body=body,
+            timeout=_REQUEST_TIMEOUT_S,
+            unavailable_hint=_UNAVAILABLE_HINT,
+        )
+        return self._record(payload)
+
+    def list_exports_page(
+        self, job_id: str, *, limit: int, page: int = 1, scope: str = SOURCE_JOB_SCOPE
+    ) -> tuple[list[dict[str, Any]], int]:
+        """One page of *job_id*'s exports (newest first) plus the total count."""
+        url = (
+            self._base_url
+            + self._path(job_id)
+            + "?"
+            + urllib.parse.urlencode({"limit": str(limit), "page": str(page), "scope": scope})
+        )
+        payload = http.request_json(
+            "GET",
+            url,
+            token=self._token,
+            api_base_url=self._base_url,
+            timeout=_REQUEST_TIMEOUT_S,
+            unavailable_hint=_UNAVAILABLE_HINT,
+        )
+        items = payload.get("items") if isinstance(payload, dict) else None
+        total = payload.get("total") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not isinstance(total, int):
+            raise ExportApiError("Malformed exports list response (no 'items'/'total').")
+        return [item for item in items if isinstance(item, dict)], total
+
+    def find_export(
+        self, job_id: str, export_job_id: str, *, scope: str = SOURCE_JOB_SCOPE
+    ) -> Optional[dict[str, Any]]:
+        """The export record for *export_job_id* under source job *job_id*.
+
+        ``None`` when the listing does not carry it — a state the caller must
+        report rather than paper over: an export that vanished from its own
+        source job's listing is a server-side inconsistency, never a reason to
+        assume any particular status.
+        """
+        page = 1
+        while page <= _LIST_MAX_PAGES:
+            items, _total = self.list_exports_page(job_id, limit=_LIST_PAGE_LIMIT, page=page, scope=scope)
+            for item in items:
+                if str(item.get("export_job_id")) == export_job_id:
+                    return self._record(item)
+            if len(items) < _LIST_PAGE_LIMIT:
+                return None
+            page += 1
+        return None
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _path(job_id: str) -> str:
+        return JOB_EXPORTS_ROUTE_TEMPLATE.format(job_id=http.quote_path_segment(job_id))
+
+    @staticmethod
+    def _record(payload: Any) -> dict[str, Any]:
+        """Validate the two fields every later decision reads.
+
+        Deliberately narrow: only ``export_job_id`` and ``status`` are checked,
+        because everything else on the record is either optional by contract
+        (``validation`` is ``null`` until the export finishes) or display text.
+        A response missing either of these cannot be polled or reported on at
+        all, so it fails here rather than as an opaque ``KeyError`` later.
+        """
+        if not isinstance(payload, dict):
+            raise ExportApiError("Malformed export record response: expected a JSON object.")
+        if not payload.get("export_job_id") or not payload.get("status"):
+            raise ExportApiError("Malformed export record response (no 'export_job_id'/'status').")
+        return payload

simulo/_client/export_bundle.py

@@ -0,0 +1,296 @@
+"""Unpack a downloaded policy-bundle archive, safely and to the contract.
+
+``simulo export`` hands the user a DIRECTORY they immediately ``cd`` into and
+run ``python verify.py`` from, so this module owns the step between "verified
+bytes on disk" and "a bundle the next command in the printed block will work
+against". Three jobs, all fail-closed:
+
+* **Extraction safety.** The archive is server-supplied. Every member name is
+  reduced and checked before a single byte is written — no absolute paths, no
+  ``..``, no drive letters or backslash separators (a Windows-hostile name is
+  still a plain name on POSIX and must be refused on both), no links or
+  devices, nothing outside the destination, and a running uncompressed-size
+  ceiling enforced DURING the copy.
+
+  What that ceiling defends is a large LEGITIMATE member — an honest
+  ``model.onnx`` or ``model.pt`` big enough to fill the user's disk from a
+  command they ran to fetch a model. It is deliberately NOT defending against
+  a forged ``file_size``: that bypass is unreachable, and this module used to
+  claim otherwise. ``zipfile`` bounds each member's decompressed output by the
+  size the header declares and raises ``BadZipFile`` on the CRC mismatch a
+  forgery produces, so a lying header cannot yield more bytes than it declared
+  (measured: a member whose declared size is understated delivers ZERO bytes,
+  not too many). Enforcing during the copy is still the right shape — it
+  bounds the bytes actually WRITTEN, which is the quantity the disk cares
+  about — but it buys no extra protection against a dishonest header.
+* **Contract conformance.** ``simulo.interfaces.platform.policy_bundle``
+  defines exactly which files a bundle holds and which of them are required.
+  An archive carrying a file outside that set, or missing one of the required
+  ones, is refused with the offending name — the local-verification guarantee
+  is a property of the FORMAT, so an archive that is not that format must not
+  be presented as one.
+* **Never destroying a good bundle for a bad archive.** Extraction happens in
+  a staging directory beside the destination and is only swapped into place
+  once the whole archive has passed. Re-running ``simulo export`` therefore
+  replaces the previous bundle atomically on success and leaves it entirely
+  untouched on failure.
+
+Names and the file set come from the contract module; nothing here is
+retyped.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+import zipfile
+from pathlib import Path, PurePosixPath
+from typing import Optional
+
+from simulo.interfaces.platform.policy_bundle import (
+    MAX_POLICY_BUNDLE_UNCOMPRESSED_BYTES,
+    POLICY_BUNDLE_ARCHIVE_NAME_TEMPLATE,
+    POLICY_BUNDLE_FILES,
+    POLICY_BUNDLE_MANIFEST_FILE,
+    POLICY_BUNDLE_REQUIRED_FILES,
+    is_supported_policy_bundle_schema,
+)
+
+#: Suffix every bundle archive carries, derived from the contract's own
+#: archive-name template rather than typed as ``".zip"``.
+BUNDLE_ARCHIVE_SUFFIX = POLICY_BUNDLE_ARCHIVE_NAME_TEMPLATE.rsplit("}", 1)[-1]
+
+#: Ceiling on the TOTAL uncompressed bytes one bundle may expand to — THE
+#: contract's own value, not a number this module chose.
+#:
+#: Binding it to :data:`MAX_POLICY_BUNDLE_UNCOMPRESSED_BYTES` is what makes the
+#: refusal below safe to ship: the exporter holds every bundle it publishes to
+#: the same budget (``simulo.backend.export.exporter``), so this command
+#: refuses only archives that are NOT conforming bundles. When the two were
+#: independent literals — and for a while the producer had no limit at all —
+#: this ceiling could refuse a bundle Simulo had happily built, validated,
+#: uploaded and billed for (issue #654, QA pass). The alias is kept as a
+#: module-level name because it is what the refusal message quotes.
+MAX_BUNDLE_UNCOMPRESSED_BYTES = MAX_POLICY_BUNDLE_UNCOMPRESSED_BYTES
+
+#: Chunk size for the size-capped member copy.
+_COPY_CHUNK_BYTES = 256 * 1024
+
+
+class PolicyBundleError(ValueError):
+    """A downloaded bundle archive that cannot be safely or faithfully unpacked.
+
+    A ``ValueError`` so the CLI's existing local-failure handling reports it as
+    one clean line; the message always names the specific member or missing
+    file, never just "invalid bundle".
+    """
+
+
+def bundle_root_dir_name(archive_name: str) -> str:
+    """The top-level directory *archive_name* must contain.
+
+    The contract pins archive and root-directory names to the same
+    ``{policy_name}-policy-{model_kind}`` stem, so the expected directory is
+    the archive filename minus its suffix — read from the templates, not
+    re-derived from a job name (which is exactly what
+    ``policy_bundle_root_dir``/``policy_bundle_archive_name`` exist to stop
+    consumers doing).
+    """
+    stem = archive_name[: -len(BUNDLE_ARCHIVE_SUFFIX)] if archive_name.endswith(BUNDLE_ARCHIVE_SUFFIX) else archive_name
+    if not stem:
+        raise PolicyBundleError(f"Bundle archive name {archive_name!r} has no directory stem.")
+    return stem
+
+
+def _member_relative_path(name: str, *, root: str) -> Optional[str]:
+    """Reduce one archive member name to its path INSIDE the bundle root.
+
+    Returns ``None`` for the root directory entry itself (which is skipped,
+    not written); raises :class:`PolicyBundleError` for anything unsafe or
+    off-contract. Every rejection names the member.
+    """
+    if "\\" in name:
+        # A backslash is a legal character in a POSIX filename, so a member
+        # named ``a\..\..\evil`` extracts to one harmless file here and to a
+        # traversal on Windows. Refuse on every platform rather than letting
+        # the archive's meaning depend on where it is unpacked.
+        raise PolicyBundleError(f"Bundle member {name!r} contains a backslash; refusing to unpack it.")
+    if name.startswith("/") or (len(name) > 1 and name[1] == ":"):
+        raise PolicyBundleError(f"Bundle member {name!r} is an absolute path; refusing to unpack it.")
+    parts = [part for part in PurePosixPath(name).parts if part not in ("", ".")]
+    if any(part == ".." for part in parts):
+        raise PolicyBundleError(f"Bundle member {name!r} escapes the bundle directory; refusing to unpack it.")
+    if not parts:
+        raise PolicyBundleError(f"Bundle member {name!r} has no usable name; refusing to unpack it.")
+    if parts[0] != root:
+        raise PolicyBundleError(
+            f"Bundle member {name!r} is outside the bundle's own directory {root!r}; refusing to unpack it."
+        )
+    inner = parts[1:]
+    if not inner:
+        return None  # the root directory entry itself
+    if len(inner) > 1 or inner[0] not in POLICY_BUNDLE_FILES:
+        raise PolicyBundleError(
+            f"Bundle member {name!r} is not part of a Simulo policy bundle "
+            f"(expected one of: {', '.join(POLICY_BUNDLE_FILES)}); refusing to unpack it."
+        )
+    return inner[0]
+
+
+def _is_link_or_special(info: zipfile.ZipInfo) -> bool:
+    """``True`` for a member that is not a plain file or directory.
+
+    ``external_attr``'s high 16 bits carry the POSIX mode when the archive was
+    written on a Unix host, and only the FILE-TYPE bits of that word decide
+    this: a symlink (``S_IFLNK``) is refused — ``zipfile`` writes its target as
+    file CONTENT, and a later reader following it leaves the bundle — as is any
+    other non-regular, non-directory type.
+
+    The type bits being ABSENT is not a signal of anything and must not be read
+    as one: ``ZipFile.writestr`` stamps a bare ``0o600`` permission word with no
+    type bits at all, so a "type must be S_IFREG" test rejects perfectly
+    ordinary archives (measured — it rejected this package's own fixtures), and
+    a zero high word means no mode was recorded at all.
+    """
+    file_type = (info.external_attr >> 16) & 0o170000
+    return file_type not in (0, 0o100000, 0o040000)
+
+
+def _copy_member(archive: zipfile.ZipFile, info: zipfile.ZipInfo, dest: Path, *, budget: int) -> int:
+    """Copy one member to *dest*, consuming at most *budget* bytes; return the size."""
+    written = 0
+    with archive.open(info) as source, open(dest, "wb") as target:
+        while True:
+            chunk = source.read(_COPY_CHUNK_BYTES)
+            if not chunk:
+                break
+            written += len(chunk)
+            if written > budget:
+                raise PolicyBundleError(
+                    f"Bundle expands past the {MAX_BUNDLE_UNCOMPRESSED_BYTES} byte unpack ceiling at member "
+                    f"{info.filename!r}; refusing to unpack it."
+                )
+            target.write(chunk)
+    return written
+
+
+def _carries_bundle_manifest(dest_dir: Path) -> bool:
+    """Is *dest_dir* POSITIVELY identified as a policy bundle?
+
+    A readable ``manifest.json`` declaring a bundle schema version this
+    software supports. Nothing weaker will do: the archive that determines
+    which NAMES land in the destination is server-supplied, so "every entry is
+    a bundle member name" is a property an attacker chooses, not evidence
+    about what was there before. The marker is content the bundle FORMAT
+    requires and a bystander directory has no reason to contain.
+    """
+    manifest_path = dest_dir / POLICY_BUNDLE_MANIFEST_FILE
+    if manifest_path.is_symlink() or not manifest_path.is_file():
+        return False
+    try:
+        parsed = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, ValueError):
+        return False
+    if not isinstance(parsed, dict):
+        return False
+    version = parsed.get("schema_version")
+    # The contract's check takes a str; a manifest that does not even carry one
+    # is not a bundle manifest, which is the same answer.
+    return isinstance(version, str) and is_supported_policy_bundle_schema(version)
+
+
+def _assert_replaceable(dest_dir: Path) -> None:
+    """Refuse unless *dest_dir* is absent or unmistakably a previous bundle.
+
+    Re-running ``simulo export`` must not fail on its own last output, and it
+    must never delete anything else. Two conditions, and the second is the one
+    that carries the guarantee:
+
+    * every entry is a bundle member file — anything else (a user's own file,
+      a subdirectory, a same-named project folder) is refused by name; and
+    * the directory carries the bundle's own marker
+      (:func:`_carries_bundle_manifest`). A name-set test alone is decided
+      entirely by names the DOWNLOADED ARCHIVE chose, so an archive whose
+      members happen to be spelled like a bundle's would make this function
+      approve deleting an existing directory that never was one. The marker is
+      the positive evidence; absence of contradiction is not evidence.
+
+    The cost is deliberate: a previous download whose ``manifest.json`` is
+    unreadable is no longer replaced automatically, and the refusal says so
+    and names the two ways out.
+    """
+    if not dest_dir.exists():
+        return
+    if dest_dir.is_symlink() or not dest_dir.is_dir():
+        raise PolicyBundleError(
+            f"{dest_dir} already exists and is not a directory. Move it aside, or pass -o DIR to unpack "
+            "somewhere else."
+        )
+    for entry in dest_dir.iterdir():
+        if entry.is_symlink() or not entry.is_file() or entry.name not in POLICY_BUNDLE_FILES:
+            raise PolicyBundleError(
+                f"{dest_dir} already exists and is not a previously downloaded bundle (it contains "
+                f"{entry.name!r}). Move it aside, or pass -o DIR to unpack somewhere else."
+            )
+    if not _carries_bundle_manifest(dest_dir):
+        raise PolicyBundleError(
+            f"{dest_dir} already exists and is not a previously downloaded bundle (no readable "
+            f"{POLICY_BUNDLE_MANIFEST_FILE} declaring a supported bundle version). Move it aside, or pass "
+            "-o DIR to unpack somewhere else."
+        )
+
+
+def _extract_into(archive_path: Path, staging_root: Path, *, root: str, archive_name: str) -> Path:
+    """Extract every member of *archive_path* under ``staging_root/root``."""
+    staged = staging_root / root
+    staged.mkdir()
+    try:
+        with zipfile.ZipFile(archive_path) as archive:
+            budget = MAX_BUNDLE_UNCOMPRESSED_BYTES
+            written: set[str] = set()
+            for info in archive.infolist():
+                if _is_link_or_special(info):
+                    raise PolicyBundleError(
+                        f"Bundle member {info.filename!r} is a link or special file; refusing to unpack it."
+                    )
+                relative = _member_relative_path(info.filename, root=root)
+                if relative is None or info.is_dir():
+                    continue
+                budget -= _copy_member(archive, info, staged / relative, budget=budget)
+                written.add(relative)
+    except zipfile.BadZipFile as exc:
+        raise PolicyBundleError(f"Bundle archive {archive_name!r} is not a readable zip archive ({exc}).") from exc
+    missing = [name for name in POLICY_BUNDLE_REQUIRED_FILES if name not in written]
+    if missing:
+        raise PolicyBundleError(f"Bundle archive {archive_name!r} is missing required file(s): {', '.join(missing)}.")
+    return staged
+
+
+def extract_policy_bundle(archive_path: Path, dest_root: Path, *, archive_name: str) -> Path:
+    """Unpack *archive_path* under *dest_root*; return the bundle directory.
+
+    The archive must contain exactly one top-level directory, named for the
+    archive itself, holding only contract-declared bundle files and all of the
+    required ones. Everything is staged in a sibling temporary directory and
+    swapped in only once the whole archive has passed, so a refused archive
+    leaves *dest_root* exactly as it found it — including any bundle a previous
+    run of this command left there.
+    """
+    root = bundle_root_dir_name(archive_name)
+    dest_dir = dest_root / root
+    _assert_replaceable(dest_dir)
+    staging_root = Path(tempfile.mkdtemp(dir=dest_root, prefix=f".{root}.", suffix=".unpack"))
+    try:
+        staged = _extract_into(archive_path, staging_root, root=root, archive_name=archive_name)
+        # Re-check immediately before the swap: the pre-check above ran before
+        # a potentially long extraction, so this is the narrow window's close,
+        # not a duplicate.
+        _assert_replaceable(dest_dir)
+        if dest_dir.exists():
+            shutil.rmtree(dest_dir)
+        os.replace(staged, dest_dir)
+    finally:
+        shutil.rmtree(staging_root, ignore_errors=True)
+    return dest_dir