engram-dynamics 0.1.0__py3-none-any.whl

engram_cli/__init__.py

@@ -0,0 +1,8 @@
+"""engram-dynamics — the `engram` command-line client for the Engram platform.
+
+Entry point is `engram_cli.main:main` (the `engram` console script). The package also
+ships the repo's `mcp_server` stdio server, so one install gives a customer both the
+CLI and their MCP server (CAAS-904).
+"""
+
+__version__ = "0.1.0"

engram_cli/api.py

@@ -0,0 +1,525 @@
+"""HTTP client for the Engram control plane.
+
+Every call goes through `Client.request`, so the error mapping lives in exactly one
+place: the server's own message is what the user sees, and the CLI never invents a
+reason of its own. Auth is `Authorization: Bearer <token>`; an `ek_`-prefixed API key
+(CAAS-801) and a session JWT are both accepted there, so the CLI does not care which
+one it is holding.
+
+API VERSION (CAAS-802). The same client speaks both contracts, chosen per profile:
+
+  legacy (the default)  the unversioned paths this CLI has always called
+  1                     the published /v1 contract — one error envelope, list
+                        responses wrapped in {"items", "next_cursor"}, and
+                        Idempotency-Key honoured on writes
+  auto                  ask the server: GET / lists `api_versions`, so a deployment
+                        that advertises "1" gets it and an older one falls back
+
+THE DEFAULT STAYS LEGACY on purpose. An installed CLI must not change behaviour
+because a server rolled forward, and a customer on an older self-hosted control plane
+has no /v1 to talk to. `engram login --api-version 1` (or `auto`) opts in, per profile.
+
+Everything below this line is version-agnostic: the commands call the same methods and
+see the same lists and the same ApiError messages either way. The two places the
+contracts differ — the error body and the list envelope — are normalised in
+`_detail_message` / `_detail_code` / `_retry_after` and in `Client._list`.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from .config import Profile
+
+# Generous enough for a control-plane call over a cold ECS task, short enough that a
+# wedged network fails the command instead of hanging a script.
+DEFAULT_TIMEOUT = 30.0
+
+# Profile values for `api_version`. "legacy" is the sentinel for the unversioned paths; anything
+# else is a version number and becomes the path prefix ("1" -> "/v1").
+LEGACY_VERSION = "legacy"
+AUTO_VERSION = "auto"
+# The contract's maximum page size (backend/app/pagination.py). Asking for it is how a client that
+# wants everything makes the fewest round trips.
+MAX_PAGE = 500
+
+
+def _items(body: Any) -> list[dict]:
+    """The rows out of a list response, whichever contract produced it."""
+    if isinstance(body, dict):
+        return body.get("items") or []
+    return body or []
+
+
+class ApiError(Exception):
+    """A request failed. `message` is already user-facing — print it and exit 1.
+
+    `code` and `retry_after` carry the structured half of the error (CAAS-111: 429
+    `ingest_quota` / `beta_limit` / `plan_limit`, 402 `trial_expired`, 503
+    `serving_unavailable`) for the few callers that need to BRANCH on a failure rather
+    than just report it — `engram push` backs off and retries a 503 upload, and stops
+    dead on a quota 429 instead of hammering the rest of a 20,000-file manifest.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status: int | None = None,
+        body: Any = None,
+        code: str | None = None,
+        retry_after: float | None = None,
+    ):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+        self.body = body
+        self.code = code
+        self.retry_after = retry_after
+
+
+class NotLoggedIn(ApiError):
+    def __init__(self, profile_name: str):
+        super().__init__(
+            f"No API key for profile '{profile_name}'. Run: engram login --profile "
+            f"{profile_name} --api-key ek_..."
+        )
+
+
+def _envelope(body: Any) -> dict | None:
+    """The /v1 error envelope, `{"error": {"code", "message", ...}}`, or None.
+
+    Checked on `code` rather than on the key alone: a legacy structured detail also uses
+    the word "error", and mistaking one for the other would print a dict at the user."""
+    if not isinstance(body, dict):
+        return None
+    error = body.get("error")
+    return error if isinstance(error, dict) and "code" in error else None
+
+
+def _validation_line(errors: Any) -> str | None:
+    """FastAPI's per-field validation list, flattened to one line. Same rendering for both
+    contracts — under /v1 the list moved into `details.errors`, its items did not change."""
+    if not isinstance(errors, list):
+        return None
+    parts = []
+    for item in errors:
+        if isinstance(item, dict) and item.get("msg"):
+            loc = ".".join(str(p) for p in (item.get("loc") or []) if p != "body")
+            parts.append(f"{loc}: {item['msg']}" if loc else str(item["msg"]))
+        else:
+            parts.append(str(item))
+    return "; ".join(parts) or None
+
+
+def _detail_message(body: Any) -> str | None:
+    """Turn any error body — either contract — into one line.
+
+    Four shapes in the wild:
+      * {"detail": "plain string"}                      — the common legacy case
+      * {"detail": {"error": "beta_limit", "message": ...}} — beta caps (429); the
+        `message` is written for the end user, so it wins over the error code.
+      * {"detail": [{"loc": [...], "msg": ...}, ...]}    — 422 validation errors.
+      * {"error": {"code": ..., "message": ..., "details": {...}}} — the /v1 envelope.
+
+    The /v1 branch is first and deliberately produces the SAME sentence the legacy branch
+    would: the envelope is a re-packaging of the same message and the same extras, so a
+    customer who switches contracts must not see the CLI start saying different things.
+    """
+    error = _envelope(body)
+    if error is not None:
+        details = error.get("details") if isinstance(error.get("details"), dict) else {}
+        message = str(error.get("message") or error.get("code") or "").strip()
+        fields = _validation_line(details.get("errors"))
+        if fields:
+            message = f"{message} {fields}".strip() if message else fields
+        upgrade = details.get("upgrade_url")
+        if upgrade and str(upgrade) not in message:
+            message = f"{message} Upgrade: {upgrade}"
+        return message or None
+
+    if not isinstance(body, dict):
+        return None
+    detail = body.get("detail")
+    if detail is None:
+        return None
+    if isinstance(detail, str):
+        return detail
+    if isinstance(detail, dict):
+        message = str(detail.get("message") or detail.get("error") or detail)
+        # A quota or plan wall is only actionable with the link out of it, and the server
+        # is the one that knows which plan page applies. Appended rather than reworded.
+        upgrade = detail.get("upgrade_url")
+        if upgrade and str(upgrade) not in message:
+            message = f"{message} Upgrade: {upgrade}"
+        return message
+    if isinstance(detail, list):
+        return _validation_line(detail)
+    return str(detail)
+
+
+def _detail_code(body: Any) -> str | None:
+    """The machine-readable error code, from either contract.
+
+    This is what `engram push` branches on — it backs off on `serving_unavailable` and stops
+    dead on `ingest_quota` — so the codes have to come out identical on both. They do: /v1
+    carries the SAME code, just at `error.code` instead of `detail.error`."""
+    error = _envelope(body)
+    if error is not None:
+        code = error.get("code")
+        return str(code) if code else None
+    if not isinstance(body, dict):
+        return None
+    detail = body.get("detail")
+    if isinstance(detail, dict):
+        code = detail.get("error") or detail.get("code")
+        return str(code) if code else None
+    return None
+
+
+def _retry_after(body: Any, resp: httpx.Response) -> float | None:
+    """Seconds to wait, from the structured detail or the standard header.
+
+    Read defensively from both places because the two sources disagree in the wild: a
+    proxy or the rate limiter may set the header while the application sets the field.
+    Either one is better than the CLI inventing a backoff of its own.
+    """
+    candidates: list[Any] = []
+    error = _envelope(body)
+    if error is not None and isinstance(error.get("details"), dict):
+        candidates.append(error["details"].get("retry_after"))
+    if isinstance(body, dict) and isinstance(body.get("detail"), dict):
+        candidates.append(body["detail"].get("retry_after"))
+    candidates.append(resp.headers.get("retry-after"))
+    for value in candidates:
+        if value is None:
+            continue
+        try:
+            return max(0.0, float(value))
+        except (TypeError, ValueError):
+            continue  # an HTTP-date Retry-After; not worth parsing for a backoff hint
+    return None
+
+
+class Client:
+    """Thin wrapper over httpx. One instance per command invocation."""
+
+    def __init__(self, profile: Profile, *, timeout: float = DEFAULT_TIMEOUT):
+        self.profile = profile
+        self.base_url = profile.api_url
+        self._timeout = timeout
+        # Resolved lazily, and once per client, because `auto` costs an extra round trip and a
+        # command that makes ten calls should pay for it no more than a command that makes one.
+        self._prefix: str | None = None
+
+    # --- plumbing ---------------------------------------------------------------
+    def _headers(self, *, auth: bool) -> dict[str, str]:
+        headers = {"Accept": "application/json"}
+        if auth:
+            if not self.profile.api_key:
+                raise NotLoggedIn(self.profile.name)
+            headers["Authorization"] = f"Bearer {self.profile.api_key}"
+        return headers
+
+    @property
+    def api_prefix(self) -> str:
+        """"" for the legacy paths, "/v1" for the published contract.
+
+        `auto` asks the server: GET / is unauthenticated and lists `api_versions`, so this works
+        before login and on a deployment that has never heard of versions (it 404s, and we fall
+        back to legacy, which is the right answer for exactly that deployment)."""
+        if self._prefix is not None:
+            return self._prefix
+        version = (self.profile.api_version or LEGACY_VERSION).strip().lower()
+        if version == AUTO_VERSION:
+            version = self._advertised_version()
+        self._prefix = "" if version == LEGACY_VERSION else f"/v{version.lstrip('v')}"
+        return self._prefix
+
+    @property
+    def resolved_version(self) -> str:
+        """The version this client actually settled on — "legacy" or a number, never "auto"."""
+        prefix = self.api_prefix
+        return prefix[2:] if prefix else LEGACY_VERSION
+
+    def _advertised_version(self) -> str:
+        """The newest version this deployment says it serves, or legacy if it says nothing.
+
+        Never raises: version discovery failing is not a reason for the user's actual command to
+        fail, and the legacy paths are the safe answer in every case where it does."""
+        try:
+            body = self.request("GET", "/", auth=False, versioned=False)
+        except ApiError:
+            return LEGACY_VERSION
+        versions = body.get("api_versions") if isinstance(body, dict) else None
+        if not isinstance(versions, list) or not versions:
+            return LEGACY_VERSION
+        return str(max(versions, key=lambda v: (len(str(v)), str(v))))
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        auth: bool = True,
+        json_body: Any = None,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+        versioned: bool = True,
+    ) -> Any:
+        """Call the API and return parsed JSON (None for a 204), or raise ApiError.
+
+        `versioned=False` is for the handful of paths that have no versioned copy and never will:
+        `/health` (the load balancer's target check) and `/` (version discovery itself)."""
+        url = f"{self.base_url}{self.api_prefix if versioned else ''}{path}"
+        try:
+            resp = httpx.request(
+                method,
+                url,
+                headers={**self._headers(auth=auth), **(headers or {})},
+                json=json_body,
+                params=params,
+                timeout=self._timeout,
+                follow_redirects=True,
+            )
+        except httpx.TimeoutException as exc:
+            raise ApiError(f"Timed out talking to {self.base_url} ({exc.__class__.__name__}).") from exc
+        except httpx.HTTPError as exc:
+            raise ApiError(f"Could not reach {self.base_url}: {exc}") from exc
+
+        if resp.status_code == 204 or not resp.content:
+            if resp.is_success:
+                return None
+
+        body: Any = None
+        try:
+            body = resp.json()
+        except ValueError:
+            body = None
+
+        if resp.is_success:
+            return body
+
+        message = _detail_message(body)
+        if message is None:
+            message = resp.text.strip() or f"HTTP {resp.status_code}"
+        if resp.status_code == 401:
+            message = f"{message} (the API key was rejected — check `engram login`)"
+        raise ApiError(
+            message,
+            status=resp.status_code,
+            body=body,
+            code=_detail_code(body),
+            retry_after=_retry_after(body, resp),
+        )
+
+    def _list(self, path: str, *, auth: bool = True, params: dict[str, Any] | None = None) -> list[dict]:
+        """A list endpoint, as a plain list, on either contract.
+
+        Legacy answers a bare array. /v1 answers `{"items": [...], "next_cursor": ...}` and pages,
+        so we FOLLOW the cursor to the end: every caller here wants the whole set (resolve_corpus
+        matches a name across it, `engram docs` slices it locally), and a CLI that silently showed
+        the first 50 of 400 documents would be worse than one that never paged at all. The page
+        size is the contract maximum, so the round trips are as few as the server allows."""
+        if not self.api_prefix:
+            return self.request("GET", path, auth=auth, params=params) or []
+
+        out: list[dict] = []
+        query = dict(params or {})
+        query.setdefault("limit", MAX_PAGE)
+        while True:
+            body = self.request("GET", path, auth=auth, params=query)
+            out.extend(_items(body))
+            cursor = body.get("next_cursor") if isinstance(body, dict) else None
+            if not cursor:
+                return out
+            query["cursor"] = cursor
+
+    # --- endpoints --------------------------------------------------------------
+    def health(self) -> Any:
+        """Liveness. Deliberately unauthenticated so `engram status` works logged out, and
+        unversioned because the load balancer's target check has no business moving."""
+        return self.request("GET", "/health", auth=False, versioned=False)
+
+    def platform_status(self) -> Any:
+        """Live serving state (CAAS-1302's GET /status, which CAAS-1303 is the query-path half of).
+
+        Needs a session since 2026-09-11 (the founder made the status page sign-in only), so the
+        profile's key rides along; `engram status` only asks when a key is configured. Cached
+        server-side, so polling this is cheap."""
+        return self.request("GET", "/status", auth=True)
+
+    def list_corpora(self) -> list[dict]:
+        return self._list("/corpora")
+
+    def create_corpus(self, name: str) -> dict:
+        return self.request("POST", "/corpora", json_body={"name": name})
+
+    def get_corpus(self, corpus_id: str) -> dict:
+        return self.request("GET", f"/corpora/{corpus_id}")
+
+    def list_documents(self, corpus_id: str, *, limit: int | None = None, offset: int = 0) -> list[dict]:
+        """Documents for a corpus.
+
+        GET /corpora/{id}/documents returns the whole list today — there is no
+        server-side paging yet (it arrives with CAAS-903), and guessing whether a
+        response was paged is not something a client can do reliably. So `--limit` and
+        `--offset` are applied here, deterministically. When the endpoint starts paging,
+        move the slice into query params; the command surface does not change.
+        """
+        docs = self._list(f"/corpora/{corpus_id}/documents")
+        if offset:
+            docs = docs[offset:]
+        if limit is not None:
+            docs = docs[:limit]
+        return docs
+
+    def list_jobs(self, corpus_id: str) -> list[dict]:
+        return self._list(f"/corpora/{corpus_id}/jobs")
+
+    # API keys (CAAS-801, built in parallel — paths are the agreed contract).
+    def list_api_keys(self) -> list[dict]:
+        return self._list("/api-keys")
+
+    def create_api_key(self, name: str, scopes: list[str], expires_at: str | None = None) -> dict:
+        body: dict[str, Any] = {"name": name, "scopes": scopes}
+        if expires_at:
+            body["expires_at"] = expires_at
+        return self.request("POST", "/api-keys", json_body=body)
+
+    def revoke_api_key(self, key_id: str) -> Any:
+        return self.request("DELETE", f"/api-keys/{key_id}")
+
+    # --- ingest contract (CAAS-106/107/109, consumed by `engram push`) ------------
+    #
+    # The three calls are one transaction in the user's head and three on the wire, on
+    # purpose: diff is cheap and answers "what actually changed", upload-urls is issued
+    # only for that delta, and commit is what makes any of it real. A run interrupted
+    # between any two of them is resumable precisely because nothing is registered until
+    # commit — the next diff sees the same work still outstanding.
+    def platform_info(self) -> dict:
+        """Region, egress guidance, and the extensions this deployment can ingest.
+
+        Unauthenticated (CAAS-608) so it also answers the pre-signup question, which is
+        why `engram push` can use it to learn the supported extensions before doing
+        anything that needs a key.
+        """
+        return self.request("GET", "/platform/info", auth=False) or {}
+
+    def diff_documents(self, corpus_id: str, files: list[dict]) -> dict:
+        """The rsync step: send the manifest, get back what is new/changed/unchanged/missing."""
+        return self.request(
+            "POST", f"/corpora/{corpus_id}/documents/diff", json_body={"files": files}
+        ) or {}
+
+    def upload_urls(self, corpus_id: str, files: list[dict]) -> list[dict]:
+        """Ask for a write target per file. Each may be a presigned bucket URL or a URL
+        on the API itself; `push.py` decides which credential to send from the origin."""
+        body = self.request(
+            "POST", f"/corpora/{corpus_id}/documents/upload-urls", json_body={"files": files}
+        ) or {}
+        return body.get("uploads") or []
+
+    def commit_documents(
+        self, corpus_id: str, files: list[dict], *, onboard: bool, delete_missing: bool
+    ) -> dict:
+        """Register the uploaded bytes and open a sync run.
+
+        `files` is the WHOLE manifest, not just the delta: `delete_missing` can only mean
+        "remove what this manifest does not mention", so the server has to see the full
+        desired state to compute a removal safely.
+        """
+        return self.request(
+            "POST",
+            f"/corpora/{corpus_id}/documents/commit",
+            json_body={"files": files, "onboard": onboard, "delete_missing": delete_missing},
+        ) or {}
+
+    def upsert_document(
+        self,
+        corpus_id: str,
+        path: str,
+        *,
+        text: str | None = None,
+        content_base64: str | None = None,
+    ) -> dict:
+        body: dict[str, Any] = {"path": path}
+        if text is not None:
+            body["text"] = text
+        if content_base64 is not None:
+            body["content_base64"] = content_base64
+        return self.request("POST", f"/corpora/{corpus_id}/documents/upsert", json_body=body) or {}
+
+    # --- sync runs (CAAS-109) -----------------------------------------------------
+    def get_sync_run(self, corpus_id: str, run_id: str) -> dict:
+        return self.request("GET", f"/corpora/{corpus_id}/sync-runs/{run_id}") or {}
+
+    def list_sync_runs(self, corpus_id: str) -> list[dict]:
+        """Run history, newest first if the server orders it; a bare list or a
+        {"runs": [...]} envelope are both accepted because either is a reasonable shape
+        and guessing wrong would break `engram status` for a cosmetic reason."""
+        if self.api_prefix:
+            return self._list(f"/corpora/{corpus_id}/sync-runs")
+        body = self.request("GET", f"/corpora/{corpus_id}/sync-runs")
+        if isinstance(body, dict):
+            return body.get("runs") or body.get("items") or []
+        return body or []
+
+    # --- sources (CAAS-601/903) ---------------------------------------------------
+    def create_source(self, corpus_id: str, body: dict) -> dict:
+        return self.request("POST", f"/corpora/{corpus_id}/sources", json_body=body) or {}
+
+    def list_sources(self, corpus_id: str) -> list[dict]:
+        return self._list(f"/corpora/{corpus_id}/sources")
+
+    def validate_source(self, corpus_id: str, source_id: str) -> dict:
+        return self.request(
+            "POST", f"/corpora/{corpus_id}/sources/{source_id}/validate"
+        ) or {}
+
+    def sync_source(self, corpus_id: str, source_id: str) -> dict:
+        """Kick a pull now instead of waiting for the schedule. Returns a sync run."""
+        return self.request("POST", f"/corpora/{corpus_id}/sources/{source_id}/sync") or {}
+
+    def delete_source(self, corpus_id: str, source_id: str) -> Any:
+        return self.request("DELETE", f"/corpora/{corpus_id}/sources/{source_id}")
+
+    def s3_setup(
+        self, *, external_id: str, bucket: str, prefix: str = "", kms_key_arn: str | None = None
+    ) -> dict:
+        """The customer-side IAM templates for one source. A pure renderer server-side,
+        so it holds no secret and needs nothing to exist first."""
+        params: dict[str, Any] = {"external_id": external_id, "bucket": bucket, "prefix": prefix}
+        if kms_key_arn:
+            params["kms_key_arn"] = kms_key_arn
+        return self.request("GET", "/sources/s3/setup", params=params) or {}
+
+
+def resolve_corpus(client: Client, ref: str) -> dict:
+    """Accept a corpus by exact id OR exact name, anywhere a corpus is named.
+
+    One list call answers both, and gives good errors: an unknown ref names the closest
+    thing the tenant actually has, and a duplicated name lists the candidate ids rather
+    than silently picking one.
+    """
+    corpora = client.list_corpora()
+    for corpus in corpora:
+        if corpus.get("id") == ref:
+            return corpus
+
+    by_name = [c for c in corpora if c.get("name") == ref]
+    if not by_name:
+        # Case-insensitive second pass — people type `engram status sales` for "Sales".
+        by_name = [c for c in corpora if (c.get("name") or "").lower() == ref.lower()]
+
+    if len(by_name) == 1:
+        return by_name[0]
+    if len(by_name) > 1:
+        candidates = "\n".join(f"  {c.get('id')}  {c.get('name')}" for c in by_name)
+        raise ApiError(
+            f"'{ref}' matches {len(by_name)} document bases. Use the id:\n{candidates}"
+        )
+
+    known = ", ".join(sorted(c.get("name", "") for c in corpora)) or "(none)"
+    raise ApiError(f"No document base named or with id '{ref}'. You have: {known}")

engram_cli/common.py

@@ -0,0 +1,63 @@
+"""The conventions every `engram` command shares, in one importable place.
+
+`main.py` owns the command surface, but `sources_cmd.py` needs the same `--profile` /
+`--json` handling and the same error-to-exit-code mapping, and importing `main` from a
+command module it registers would be circular. So the four pieces both need live here:
+the merged option object, the two shared `typer.Option` declarations, the error decorator,
+and the client factory.
+
+Behaviour is unchanged from where it started in `main.py` — every command still accepts
+its flags before OR after the subcommand, and still exits 1 with the SERVER's message on
+stderr rather than a reason the CLI invented.
+"""
+from __future__ import annotations
+
+import functools
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import typer
+
+from . import output
+from .api import ApiError, Client
+from .config import ConfigError, load_profile
+
+
+@dataclass
+class CommonOpts:
+    profile: str | None = None
+    json_out: bool = False
+
+
+PROFILE_OPT = typer.Option(None, "--profile", "-p", help="Config profile to use (default: 'default').")
+JSON_OPT = typer.Option(False, "--json", help="Emit JSON on stdout for scripting.")
+
+
+def opts(ctx: typer.Context, profile: str | None, json_out: bool) -> CommonOpts:
+    """Merge the group-level flags (`engram --json status`) with the command-level ones
+    (`engram status --json`). Either position works; the command-level one wins."""
+    base: CommonOpts = ctx.obj if isinstance(ctx.obj, CommonOpts) else CommonOpts()
+    return CommonOpts(profile=profile or base.profile, json_out=bool(json_out or base.json_out))
+
+
+def handle_errors(func: Callable) -> Callable:
+    """Turn an ApiError/ConfigError into `error: <server message>` on stderr and exit 1.
+
+    Applied per command rather than around the whole app so the behaviour is identical
+    when a test drives a command through CliRunner.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except (ApiError, ConfigError) as exc:
+            output.error(str(exc))
+            raise typer.Exit(code=1) from exc
+
+    return wrapper
+
+
+def client(options: CommonOpts) -> Client:
+    return Client(load_profile(options.profile))