simulo 0.26.0__py3-none-any.whl

simulo/_client/asset_api.py

@@ -0,0 +1,517 @@
+"""HTTP client for the asset-catalog publish/validate surface (stdlib only).
+
+Implements the client half of ``simulo.interfaces.platform.asset_catalog``'s
+publish flow — initiate → (multipart|single-PUT) upload → complete →
+poll/validate — against the FROZEN v1 customer contract. Shares the request
+plumbing (structured errors, the https-when-token guard, the foreign-host
+bearer rule) with every other client via ``http.py``.
+
+**Upload orchestration** (:func:`upload_archive`) is resume-first: it reads the
+server's upload-state (S3 ListParts passthrough) BEFORE sending bytes, skips
+parts that already landed with the right size and a matching etag, uploads the
+rest against batch presigns (the ``{"parts": [...]}`` envelope), and completes
+with the etags a FINAL upload-state read reports — the one source both storage
+backends (S3 and LocalStorage) agree on, so the client never has to parse a
+PUT response header. A single-PUT upload (``upload_id == null`` in the initiate
+envelope — the frozen null-not-empty-string shape) PUTs the whole archive to
+the one presigned URL and completes with its own sha256 as the (unchecked on
+this path) etag.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import urllib.parse
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+from simulo._client import http
+from simulo.interfaces.platform.asset_catalog import (
+    ASSET_ROUTE_TEMPLATE,
+    ASSET_VERSION_BY_NUMBER_ROUTE_TEMPLATE,
+    ASSET_VERSION_DEPRECATE_ROUTE_TEMPLATE,
+    ASSET_VERSION_DOWNLOAD_ROUTE_TEMPLATE,
+    ASSET_VERSION_REACTIVATE_ROUTE_TEMPLATE,
+    ASSET_VERSION_REPORT_ROUTE_TEMPLATE,
+    ASSET_VERSION_ROUTE_TEMPLATE,
+    ASSET_VERSION_UPLOAD_COMPLETE_ROUTE_TEMPLATE,
+    ASSET_VERSION_UPLOAD_PARTS_ROUTE_TEMPLATE,
+    ASSET_VERSION_UPLOAD_ROUTE_TEMPLATE,
+    ASSET_VERSION_VALIDATE_ROUTE_TEMPLATE,
+    ASSET_VERSIONS_ROUTE_TEMPLATE,
+    ASSETS_ROUTE,
+)
+
+_REQUEST_TIMEOUT_S = 10.0  # every outbound call has an explicit timeout (NFR)
+#: Per-part PUT timeout — a part is up to 64 MiB (ASSET_UPLOAD_PART_BYTES).
+_PART_UPLOAD_TIMEOUT_S = 300.0
+#: Presign-batch ceiling per request (mirrors the CP's request-envelope cap).
+_MAX_PART_NUMBERS_PER_BATCH = 1000
+#: Whole-archive presigned GET timeout (``simulo asset get``) — a SINGLE
+#: request regardless of archive size (a presigned S3 GET has no multipart-
+#: download equivalent the way the upload side does), so this must
+#: comfortably cover the :data:`~simulo.interfaces.platform.asset_catalog.
+#: MAX_ASSET_VERSION_BYTES` (10 GiB) ceiling on a slow connection. Matches
+#: the magnitude of ``cli.py``'s own big-cloud-operation windows
+#: (``_ASSET_VALIDATION_WINDOW_S``), not the much smaller
+#: ``submit_api._DOWNLOAD_TIMEOUT_S`` (60s) sized for job models/recordings,
+#: which are typically orders of magnitude smaller than an asset package.
+_ARCHIVE_DOWNLOAD_TIMEOUT_S = 1800.0
+#: Page size for ``list_assets``'s internal pagination loop — mirrors
+#: ``submit_api.py``'s ``_LIST_PAGE_LIMIT`` (also the server's
+#: ``MAX_PAGE_LIMIT``, ``jobs/constants.py``), so a full catalog listing
+#: completes in the fewest round trips the server allows.
+_LIST_PAGE_LIMIT = 100
+#: Defense-in-depth cap on how many pages ``list_assets`` will follow —
+#: mirrors ``submit_api.py``'s ``_LIST_MAX_PAGES`` (an org's catalog is
+#: bounded by its storage quota in practice, never truly unbounded, but a
+#: cap keeps a pathological response from looping forever).
+_LIST_MAX_PAGES = 1000
+
+_UNAVAILABLE_HINT = "Check SIMULO_API_URL / SIMULO_ENV, and that you are logged in (`simulo login`)."
+
+
+class AssetApiError(http.HttpError):
+    """Base error for the asset-catalog API surface."""
+
+
+class AssetApiUnavailable(AssetApiError):
+    """The control plane cannot be reached at all."""
+
+
+class AssetApiHTTPError(AssetApiError):
+    """A structured non-2xx from the asset surface (code/message/status kept)."""
+
+    def __init__(self, status: int, code: str, message: str, *, retry_after: Optional[int] = None) -> None:
+        super().__init__(f"{code}: {message}")
+        self.status = status
+        self.code = code
+        self.message = message
+        self.retry_after = retry_after
+
+
+def _translate(exc: http.HttpError) -> AssetApiError:
+    if isinstance(exc, http.HttpHTTPError):
+        return AssetApiHTTPError(exc.status, exc.code, exc.message, retry_after=exc.retry_after)
+    if isinstance(exc, http.HttpUnavailable):
+        return AssetApiUnavailable(str(exc))
+    return AssetApiError(str(exc))
+
+
+class AssetApiClient:
+    """Bearer-authenticated client for organization catalog operations and reads."""
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        token: Optional[str] = None,
+    ) -> None:
+        if not base_url.startswith(("http://", "https://")):
+            raise AssetApiError(f"API base URL must be an http(s) URL, got {base_url!r}.")
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+
+    @property
+    def base_url(self) -> str:
+        return self._base_url
+
+    # ------------------------------------------------------------------
+    # Publish flow
+    # ------------------------------------------------------------------
+
+    def initiate(self, kind: str, name: str, body: dict[str, Any]) -> dict[str, Any]:
+        path = ASSET_VERSIONS_ROUTE_TEMPLATE.format(
+            kind=http.quote_path_segment(kind), name=http.quote_path_segment(name)
+        )
+        payload = self._request("POST", path, json_body=body)
+        if not isinstance(payload, dict) or not payload.get("version_id"):
+            raise AssetApiError(f"Malformed initiate response: {payload!r}")
+        return payload
+
+    def presign_parts(self, version_id: str, part_numbers: list[int]) -> list[dict[str, Any]]:
+        path = ASSET_VERSION_UPLOAD_PARTS_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("POST", path, json_body={"part_numbers": part_numbers})
+        parts = payload.get("parts") if isinstance(payload, dict) else None
+        if not isinstance(parts, list):
+            raise AssetApiError(f"Malformed part-presign response (no 'parts' array): {payload!r}")
+        return [part for part in parts if isinstance(part, dict)]
+
+    def get_upload_state(self, version_id: str) -> dict[str, Any]:
+        """Resume state for an in-progress organization upload."""
+        path = ASSET_VERSION_UPLOAD_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("GET", path)
+        if not isinstance(payload, dict):
+            raise AssetApiError(f"Malformed upload-state response: {payload!r}")
+        return payload
+
+    def complete(self, version_id: str, parts: list[dict[str, Any]]) -> dict[str, Any]:
+        path = ASSET_VERSION_UPLOAD_COMPLETE_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("POST", path, json_body={"parts": parts})
+        if not isinstance(payload, dict) or not payload.get("validation_job_id"):
+            raise AssetApiError(f"Malformed upload-complete response: {payload!r}")
+        return payload
+
+    def revalidate(self, version_id: str, runtime: Optional[str]) -> dict[str, Any]:
+        path = ASSET_VERSION_VALIDATE_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        body: dict[str, Any] = {}
+        if runtime is not None:
+            body["runtime"] = runtime
+        payload = self._request("POST", path, json_body=body)
+        if not isinstance(payload, dict) or not payload.get("validation_job_id"):
+            raise AssetApiError(f"Malformed revalidate response: {payload!r}")
+        return payload
+
+    # ------------------------------------------------------------------
+    # Reads (polling + ref → version resolution)
+    # ------------------------------------------------------------------
+
+    def get_asset(self, kind: str, name: str, *, publisher: Optional[str] = None) -> dict[str, Any]:
+        path = ASSET_ROUTE_TEMPLATE.format(kind=http.quote_path_segment(kind), name=http.quote_path_segment(name))
+        query = {"publisher": publisher} if publisher else None
+        payload = self._request("GET", path, query=query)
+        if not isinstance(payload, dict) or not payload.get("asset_id"):
+            raise AssetApiError(f"Malformed asset response: {payload!r}")
+        return payload
+
+    def get_version(self, version_id: str) -> dict[str, Any]:
+        path = ASSET_VERSION_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("GET", path)
+        if not isinstance(payload, dict) or not payload.get("version_id"):
+            raise AssetApiError(f"Malformed asset-version response: {payload!r}")
+        return payload
+
+    def get_report(self, version_id: str) -> dict[str, Any]:
+        """The stored validation/conversion report (customer route only)."""
+        path = ASSET_VERSION_REPORT_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("GET", path)
+        if not isinstance(payload, dict):
+            raise AssetApiError(f"Malformed report response: {payload!r}")
+        return payload
+
+    # ------------------------------------------------------------------
+    # list / search (PR-8) — ``scope=global``/``all``
+    # is still a customer read available to any authenticated user for global
+    # rows use the same read route as organization rows.
+    # ------------------------------------------------------------------
+
+    def list_assets(
+        self,
+        *,
+        scope: str,
+        kind: Optional[str] = None,
+        q: Optional[str] = None,
+        tag: Optional[str] = None,
+        include_deprecated: bool = False,
+    ) -> list[dict[str, Any]]:
+        """Every :class:`~simulo.interfaces.platform.asset_catalog.AssetRecord`
+        row matching the filters — pages internally (the
+        ``submit_api.list_recordings``/``list_models`` pattern) so a caller
+        never has to drive ``?page=``/``?limit=`` itself. ``include_deprecated``
+        (``simulo asset list --deprecated``) additionally surfaces assets whose
+        latest live version is deprecated (``latest_version_status='deprecated'``)."""
+        query_base: dict[str, str] = {"scope": scope, "limit": str(_LIST_PAGE_LIMIT)}
+        if kind:
+            query_base["kind"] = kind
+        if q:
+            query_base["q"] = q
+        if tag:
+            query_base["tag"] = tag
+        if include_deprecated:
+            query_base["include_deprecated"] = "true"
+        items: list[dict[str, Any]] = []
+        page = 1
+        while page <= _LIST_MAX_PAGES:
+            payload = self._request("GET", ASSETS_ROUTE, query={**query_base, "page": str(page)})
+            page_items = payload.get("items") if isinstance(payload, dict) else None
+            if not isinstance(page_items, list):
+                raise AssetApiError(f"Malformed asset list response: {payload!r}")
+            items.extend(item for item in page_items if isinstance(item, dict))
+            if len(page_items) < _LIST_PAGE_LIMIT:
+                return items
+            page += 1
+        return items
+
+    def list_assets_page(
+        self,
+        *,
+        scope: str,
+        kind: Optional[str] = None,
+        q: Optional[str] = None,
+        tag: Optional[str] = None,
+        include_deprecated: bool = False,
+        limit: int,
+    ) -> tuple[list[dict[str, Any]], int]:
+        """Page 1 (up to *limit* rows) of :meth:`list_assets`'s same query,
+        plus the total matching-row count.
+
+        The single-request counterpart to :meth:`list_assets` (which loops
+        every page) — backs the CLI's default recent-N ``asset list`` view /
+        ``--limit N``, so a bounded view never pays for pages it will not
+        display. ``search`` keeps using :meth:`list_assets` unchanged."""
+        query: dict[str, str] = {"scope": scope, "limit": str(limit), "page": "1"}
+        if kind:
+            query["kind"] = kind
+        if q:
+            query["q"] = q
+        if tag:
+            query["tag"] = tag
+        if include_deprecated:
+            query["include_deprecated"] = "true"
+        payload = self._request("GET", ASSETS_ROUTE, query=query)
+        items = payload.get("items") if isinstance(payload, dict) else None
+        total = payload.get("total") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not isinstance(total, int):
+            raise AssetApiError(f"Malformed asset list response: {payload!r}")
+        return [item for item in items if isinstance(item, dict)], total
+
+    # ------------------------------------------------------------------
+    # get (PR-8) — the frozen contract exposes no administrative download
+    # mirror. A global asset's refusal (403
+    # ``not_exportable``) comes from the SAME customer route every
+    # org download uses, not a separate privileged path.
+    # ------------------------------------------------------------------
+
+    def get_download_url(self, version_id: str) -> dict[str, Any]:
+        """``GET .../download`` — a presigned archive GET, or a structured
+        403 ``not_exportable`` (global asset) / 409 ``asset_version_not_downloadable``
+        (not yet published) raised as :class:`AssetApiHTTPError`."""
+        path = ASSET_VERSION_DOWNLOAD_ROUTE_TEMPLATE.format(version_id=http.quote_path_segment(version_id))
+        payload = self._request("GET", path)
+        if not isinstance(payload, dict) or not payload.get("url"):
+            raise AssetApiError(f"Malformed asset download-link response: {payload!r}")
+        return payload
+
+    def download_bytes(self, url: str) -> bytes:
+        """GET the presigned archive bytes. The foreign-host rule applies
+        exactly like :meth:`upload_part_bytes`: the bearer rides only when
+        *url*'s host equals the API host (a presigned S3 GET carries its own
+        auth in the query string)."""
+        try:
+            return http.request_bytes(
+                "GET",
+                url,
+                token=self._token,
+                api_base_url=self._base_url,
+                timeout=_ARCHIVE_DOWNLOAD_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+            )
+        except http.HttpError as exc:
+            raise _translate(exc) from exc
+
+    # ------------------------------------------------------------------
+    # deprecate / delete (PR-8) — keyed by (kind, name, vN), never the
+    # opaque version_id (the "{vN} is a typed placeholder" PR-1 resolution:
+    # ``.format(version=<int>)``, never a pre-formatted "v3" string).
+    # ------------------------------------------------------------------
+
+    def deprecate_version_by_number(self, kind: str, name: str, version: int) -> dict[str, Any]:
+        path = ASSET_VERSION_DEPRECATE_ROUTE_TEMPLATE.format(
+            kind=http.quote_path_segment(kind), name=http.quote_path_segment(name), version=version
+        )
+        payload = self._request("POST", path)
+        if not isinstance(payload, dict):
+            raise AssetApiError(f"Malformed deprecate response: {payload!r}")
+        return payload
+
+    def reactivate_version_by_number(self, kind: str, name: str, version: int) -> dict[str, Any]:
+        """``POST .../reactivate`` — the inverse of :meth:`deprecate_version_by_number`:
+        un-deprecate a version (``deprecated`` → ``published``, back in
+        list/search). Idempotent on an already-published version; the server
+        refuses a ``failed``/``uploading``/``validating`` version (409). A
+        ``deleted`` version is never reachable — it 404s (filtered out of the
+        by-number lookup), not this 409."""
+        path = ASSET_VERSION_REACTIVATE_ROUTE_TEMPLATE.format(
+            kind=http.quote_path_segment(kind), name=http.quote_path_segment(name), version=version
+        )
+        payload = self._request("POST", path)
+        if not isinstance(payload, dict):
+            raise AssetApiError(f"Malformed reactivate response: {payload!r}")
+        return payload
+
+    def delete_version_by_number(self, kind: str, name: str, version: int, *, force: bool = False) -> dict[str, Any]:
+        """``DELETE`` a version. ``force=True`` sends ``?force=true`` — the
+        override that detaches referencing job pins (tombstoning them) and
+        hard-deletes an ORG version even when a past job used it. The server
+        refuses ``force`` on a global/passthrough version (403)."""
+        path = ASSET_VERSION_BY_NUMBER_ROUTE_TEMPLATE.format(
+            kind=http.quote_path_segment(kind), name=http.quote_path_segment(name), version=version
+        )
+        query = {"force": "true"} if force else None
+        payload = self._request("DELETE", path, query=query)
+        if not isinstance(payload, dict):
+            raise AssetApiError(f"Malformed delete response: {payload!r}")
+        return payload
+
+    # ------------------------------------------------------------------
+    # Raw part upload
+    # ------------------------------------------------------------------
+
+    def upload_part_bytes(self, url: str, data: bytes) -> Optional[str]:
+        """PUT one part's bytes to its presigned URL; return the part's etag.
+
+        The bearer rides ONLY when the URL's host is the API host (LocalStorage
+        presigns point back at the control plane; S3 presigns carry their own
+        SigV4 auth in the query string — the foreign-host rule). The presigned
+        URL is the credential for a part PUT.
+
+        The etag comes from the ``ETag`` response header (S3's convention) or,
+        failing that, the JSON response body's ``etag`` field (the LocalStorage
+        part route's shape). ``None`` when neither is present.
+        """
+        captured: dict[str, str] = {}
+
+        def _capture(headers: Any) -> None:
+            etag = headers.get("ETag") if headers is not None else None
+            if etag:
+                captured["etag"] = str(etag).strip('"')
+
+        try:
+            raw = http.request_bytes(
+                "PUT",
+                url,
+                token=self._token,
+                api_base_url=self._base_url,
+                body=data,
+                content_type="application/octet-stream",
+                timeout=_PART_UPLOAD_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+                on_response_headers=_capture,
+            )
+        except http.HttpError as exc:
+            raise _translate(exc) from exc
+        if "etag" in captured:
+            return captured["etag"]
+        try:
+            body = json.loads(raw.decode("utf-8")) if raw else None
+        except ValueError:
+            body = None
+        if isinstance(body, dict) and body.get("etag"):
+            return str(body["etag"])
+        return None
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json_body: Optional[dict[str, Any]] = None,
+        query: Optional[dict[str, str]] = None,
+    ) -> Any:
+        url = self._base_url + path
+        if query:
+            url += "?" + urllib.parse.urlencode(query)
+        try:
+            return http.request_json(
+                method,
+                url,
+                token=self._token,
+                api_base_url=self._base_url,
+                json_body=json_body,
+                timeout=_REQUEST_TIMEOUT_S,
+                unavailable_hint=_UNAVAILABLE_HINT,
+            )
+        except http.HttpError as exc:
+            raise _translate(exc) from exc
+
+
+# ---------------------------------------------------------------------------
+# Upload orchestration — resume-aware multipart / single-PUT
+# ---------------------------------------------------------------------------
+
+
+def _read_chunk(tar_path: Path, part_number: int, part_size: int) -> bytes:
+    with tar_path.open("rb") as fh:
+        fh.seek((part_number - 1) * part_size)
+        return fh.read(part_size)
+
+
+def _chunk_matches_landed(chunk: bytes, landed: dict[str, Any]) -> bool:
+    """Trust a landed part only when its size matches AND its etag matches the
+    chunk under one of the two digest conventions in play (LocalStorage stages
+    parts with a sha256 etag; S3's single-part etag is the md5 hex). Anything
+    else re-uploads — an idempotent overwrite, never a wrong keep."""
+    if landed.get("size_bytes") != len(chunk):
+        return False
+    etag = str(landed.get("etag") or "").strip('"').lower()
+    if not etag:
+        return False
+    return etag in (
+        hashlib.sha256(chunk).hexdigest(),
+        hashlib.md5(chunk, usedforsecurity=False).hexdigest(),
+    )
+
+
+def upload_archive(
+    client: AssetApiClient,
+    *,
+    version_id: str,
+    upload: dict[str, Any],
+    tar_path: Path,
+    archive_sha256: str,
+    echo: Callable[[str], None],
+) -> tuple[dict[str, Any], int]:
+    """Drive the whole upload for one initiate ``upload`` envelope; complete it.
+
+    Returns ``(complete_response, skipped_parts)`` — ``skipped_parts`` is how
+    many parts had already landed (the resumable-upload disclosure). Raises
+    :class:`AssetApiError` on any failure — nothing is swallowed.
+    """
+    part_size = int(upload["part_size"])
+    part_count = int(upload["part_count"])
+    upload_id = upload.get("upload_id")
+
+    if upload_id is None:
+        # Single-PUT fast path: one presigned URL, the whole archive, and a
+        # complete whose etag is unchecked on this path (sent honestly anyway).
+        specs = client.presign_parts(version_id, [1])
+        if not specs:
+            raise AssetApiError("The server returned no presigned URL for the upload.")
+        etag = client.upload_part_bytes(str(specs[0]["url"]), tar_path.read_bytes())
+        completed = [{"part_number": 1, "etag": etag or archive_sha256}]
+        return client.complete(version_id, completed), 0
+
+    landed_by_number: dict[int, dict[str, Any]] = {}
+    state = client.get_upload_state(version_id)
+    for part in state.get("parts") or []:
+        if isinstance(part, dict) and isinstance(part.get("part_number"), int):
+            landed_by_number[part["part_number"]] = part
+
+    #: part_number -> etag for the complete call. Pre-seeded from the resume
+    #: state for skipped parts; captured from each PUT response otherwise.
+    etags: dict[int, str] = {}
+    needed: list[int] = []
+    skipped = 0
+    for part_number in range(1, part_count + 1):
+        landed = landed_by_number.get(part_number)
+        if landed is not None and _chunk_matches_landed(_read_chunk(tar_path, part_number, part_size), landed):
+            skipped += 1
+            etags[part_number] = str(landed.get("etag") or "")
+        else:
+            needed.append(part_number)
+    if skipped:
+        echo(f"  resuming upload — {skipped} of {part_count} part(s) already uploaded")
+
+    uploaded = 0
+    for start in range(0, len(needed), _MAX_PART_NUMBERS_PER_BATCH):
+        batch = needed[start : start + _MAX_PART_NUMBERS_PER_BATCH]
+        specs_by_number = {int(spec["part_number"]): spec for spec in client.presign_parts(version_id, batch)}
+        for part_number in batch:
+            spec = specs_by_number.get(part_number)
+            if spec is None:
+                raise AssetApiError(f"The server did not presign part {part_number}.")
+            chunk = _read_chunk(tar_path, part_number, part_size)
+            etag = client.upload_part_bytes(str(spec["url"]), chunk)
+            # LocalStorage's staged-part etag is the part's sha256; fall back to
+            # it when neither the header nor the body carried one.
+            etags[part_number] = etag or hashlib.sha256(chunk).hexdigest()
+            uploaded += 1
+            echo(f"  uploaded part {skipped + uploaded}/{part_count}")
+
+    completed = [{"part_number": n, "etag": etags[n]} for n in range(1, part_count + 1)]
+    return client.complete(version_id, completed), skipped