gi-ingest 0.2.0__py3-none-any.whl

gi_ingest/__init__.py

@@ -0,0 +1,10 @@
+"""The vendor offload CLI."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("gi-ingest")
+except PackageNotFoundError:  # running from a source checkout, not installed
+    __version__ = "0.0.0+dev"
+
+__all__ = ["__version__"]

gi_ingest/api.py

@@ -0,0 +1,145 @@
+"""
+Client for the GILabs vendor ingest API.
+
+Authenticates with a ``gik_`` key on ``X-API-Key``. There is no organization
+header: the key is bound to one vendor, so there is no tenant to select.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from gi_ingest import __version__
+
+DEFAULT_API_URL = "https://gilabs-data-center-production.up.railway.app"
+
+# Generous: seal validates a whole manifest against a bucket listing, which on a
+# large delivery is not instant. Everything else is fast.
+_TIMEOUT = httpx.Timeout(60.0, connect=15.0)
+
+
+class ApiError(RuntimeError):
+    def __init__(self, status: int, detail: str) -> None:
+        super().__init__(f"[{status}] {detail}")
+        self.status = status
+        self.detail = detail
+
+
+@dataclass(frozen=True)
+class ScopedCredentials:
+    access_key_id: str
+    secret_access_key: str
+    session_token: str
+    expiration: str
+    region: str
+    bucket: str
+    prefix: str
+    s3_uri: str
+
+
+class IngestClient:
+    def __init__(self, base_url: str, api_key: str) -> None:
+        self.base = base_url.rstrip("/")
+        self._client = httpx.Client(
+            headers={
+                "X-API-Key": api_key,
+                "Content-Type": "application/json",
+                # Both, because they answer different questions: the agent
+                # string is what shows up in a proxy log, and the explicit
+                # header is what a server can compare without parsing.
+                "User-Agent": f"gi-ingest/{__version__}",
+                "X-Client-Version": __version__,
+            },
+            timeout=_TIMEOUT,
+        )
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> IngestClient:
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self.close()
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        try:
+            r = self._client.request(method, f"{self.base}/api/v1{path}", **kwargs)
+        except httpx.HTTPError as exc:
+            raise ApiError(0, f"network error: {exc}") from exc
+        if r.status_code >= 400:
+            detail = r.text
+            with contextlib.suppress(Exception):  # non-JSON error body
+                detail = r.json().get("detail", detail)
+            raise ApiError(r.status_code, str(detail))
+        if r.status_code == 204 or not r.content:
+            return None
+        return r.json()
+
+    # ---- identity + destinations ----
+
+    def whoami(self) -> dict:
+        return self._request("GET", "/ingest/whoami")
+
+    def destinations(self) -> list[dict]:
+        return self._request("GET", "/ingest/destinations")["destinations"]
+
+    def check_episodes(self, episode_ids: list[str]) -> dict[str, dict]:
+        """Which of these episodes the platform already has.
+
+        Called at stage time, before anything is uploaded: on a bandwidth-bound
+        overnight run, transferring a card a colleague already delivered is the
+        expensive mistake. Returns a map keyed by episode id.
+        """
+        if not episode_ids:
+            return {}
+        out: dict[str, dict] = {}
+        # The endpoint caps a request at 1000 ids.
+        for i in range(0, len(episode_ids), 1000):
+            chunk = episode_ids[i : i + 1000]
+            body = {"episodes": [{"episode_id": e} for e in chunk]}
+            resp = self._request("POST", "/ingest/episodes/check", json=body)
+            for result in resp["results"]:
+                out[result["episode_id"]] = result
+        return out
+
+    # ---- batches ----
+
+    def open_batch(self, *, project_id: str | None, device_kind: str | None) -> dict:
+        body: dict[str, Any] = {}
+        if project_id:
+            body["project_id"] = project_id
+        if device_kind:
+            body["device_kind"] = device_kind
+        return self._request("POST", "/ingest/batches", json=body)
+
+    def refresh_credentials(self, batch_id: str) -> ScopedCredentials:
+        raw = self._request("POST", f"/ingest/batches/{batch_id}/credentials")
+        return ScopedCredentials(**{k: raw[k] for k in ScopedCredentials.__annotations__})
+
+    def seal_batch(self, batch_id: str, manifest: dict) -> dict:
+        return self._request(
+            "POST", f"/ingest/batches/{batch_id}/seal", json={"manifest": manifest}
+        )
+
+    def get_batch(self, batch_id: str) -> dict:
+        return self._request("GET", f"/ingest/batches/{batch_id}")
+
+    def list_batches(self, *, status: str | None = None, limit: int = 50) -> list[dict]:
+        params: dict[str, Any] = {"limit": limit}
+        if status:
+            params["status"] = status
+        return self._request("GET", "/ingest/batches", params=params)["batches"]
+
+    def abandon_batch(self, batch_id: str) -> dict:
+        return self._request("DELETE", f"/ingest/batches/{batch_id}")
+
+
+def credentials_from_open(payload: dict) -> ScopedCredentials:
+    """Pull the credentials out of an open-batch response."""
+    raw = payload["credentials"]
+    return ScopedCredentials(**{k: raw[k] for k in ScopedCredentials.__annotations__})

gi_ingest/config.py

@@ -0,0 +1,129 @@
+"""Credential and configuration storage: ``~/.config/gi-ingest/``."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+
+from gi_ingest.api import DEFAULT_API_URL
+
+CONFIG_DIR = Path.home() / ".config" / "gi-ingest"
+CONFIG_FILE = CONFIG_DIR / "config.json"
+
+DEFAULT_STAGING = Path.home() / "gilabs-staging"
+DEFAULT_LEDGER = Path.home() / ".local" / "state" / "gi-ingest" / "ledger.db"
+
+# Where the tool kept these before it was renamed from gilabs-ingest.
+_LEGACY_CONFIG_DIR = Path.home() / ".config" / "gilabs-ingest"
+_LEGACY_STATE_DIR = Path.home() / ".local" / "state" / "gilabs-ingest"
+
+
+def adopt_legacy_paths() -> None:
+    """Move an older install's config and ledger to the new names.
+
+    The rename would otherwise orphan both, and the ledger is not a cache: it
+    holds which episodes are staged, which are in flight, and which the server
+    has confirmed. Losing it mid-round means re-staging cards that were already
+    copied, and cards that can no longer be reclaimed because nothing remembers
+    they were delivered.
+
+    That matters more than usual right now, because the version floor makes
+    upgrading mandatory rather than optional — an upgrade that silently drops
+    a manager's queue would be a bad trade for a shorter name.
+
+    Moves rather than copies, so there is one source of truth afterwards, and
+    only when the new location does not already exist. Best-effort: a failure
+    here leaves the old files untouched and costs a `login`, not data.
+    """
+    pairs = (
+        (_LEGACY_CONFIG_DIR, CONFIG_DIR),
+        (_LEGACY_STATE_DIR, DEFAULT_LEDGER.parent),
+    )
+    for legacy, current in pairs:
+        if not legacy.is_dir() or current.exists():
+            continue
+        try:
+            current.parent.mkdir(parents=True, exist_ok=True)
+            legacy.rename(current)
+        except OSError:
+            # A different filesystem, a permission, a race with another run.
+            # The old path stays put and the user re-runs `login`.
+            pass
+
+
+@dataclass
+class Config:
+    api_url: str = DEFAULT_API_URL
+    api_key: str | None = None
+    staging_dir: Path = DEFAULT_STAGING
+    ledger_path: Path = DEFAULT_LEDGER
+    project_id: str | None = None
+
+    @property
+    def is_authenticated(self) -> bool:
+        return bool(self.api_key)
+
+
+def _follow_moved_state(path: Path) -> Path:
+    """Repoint a stored path that :func:`adopt_legacy_paths` moved out from under.
+
+    ``login`` writes the ledger path into config.json, so moving the directory
+    is only half the job — the recorded path still names the old location, and
+    the CLI would helpfully create a brand new empty ledger there. That is the
+    precise failure the move exists to prevent: a manager mid-round would find
+    nothing staged, nothing in flight, and no card reclaimable.
+
+    Only rewrites paths that sit under the legacy directory. Anything a manager
+    pointed somewhere deliberate is left where they put it.
+    """
+    try:
+        relative = path.relative_to(_LEGACY_STATE_DIR)
+    except ValueError:
+        return path
+    return DEFAULT_LEDGER.parent / relative
+
+
+def load() -> Config:
+    adopt_legacy_paths()
+    if not CONFIG_FILE.exists():
+        return Config()
+    try:
+        raw = json.loads(CONFIG_FILE.read_text())
+    except (OSError, json.JSONDecodeError):
+        # A corrupt config reads as "not configured" rather than a traceback —
+        # the fix is `gi-ingest login`, and that is what the caller is told.
+        return Config()
+    return Config(
+        api_url=raw.get("api_url") or DEFAULT_API_URL,
+        api_key=raw.get("api_key"),
+        staging_dir=Path(raw["staging_dir"]) if raw.get("staging_dir") else DEFAULT_STAGING,
+        ledger_path=_follow_moved_state(
+            Path(raw["ledger_path"]) if raw.get("ledger_path") else DEFAULT_LEDGER
+        ),
+        project_id=raw.get("project_id"),
+    )
+
+
+def save(config: Config) -> None:
+    """Persist config, 0600. The API key lives here in plaintext.
+
+    Same posture as the AWS CLI and `gidata`: a file only the user can read.
+    Worth knowing rather than assuming — the key is a bearer credential for the
+    vendor's whole upload surface, so a shared laptop should use a per-machine
+    key that can be revoked on its own.
+    """
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "api_url": config.api_url,
+        "api_key": config.api_key,
+        "staging_dir": str(config.staging_dir),
+        "ledger_path": str(config.ledger_path),
+        "project_id": config.project_id,
+    }
+    tmp = CONFIG_FILE.with_suffix(".tmp")
+    tmp.write_text(json.dumps(payload, indent=2) + "\n")
+    os.chmod(tmp, stat.S_IRUSR | stat.S_IWUSR)
+    tmp.replace(CONFIG_FILE)

gi_ingest/identity.py

@@ -0,0 +1,74 @@
+"""
+Episode identity — computed client-side so a dedup check costs a request, not an
+upload.
+
+``derive_episode_id`` must stay byte-identical to the worker's version in
+data-processing-services (``worker/ego/preprocess/concat.py``). It is the uuid5
+that becomes ``episodes.id`` for pipeline episodes, so a drift here would make
+every dedup check miss and every re-upload look novel — silently, and only
+visible as duplicated storage.
+
+There is no import to share: the worker lives in another repository and this
+package ships to vendors. The contract is pinned by ``tests/test_identity.py``,
+which asserts the exact uuid5 for a known triple.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+
+# NAMESPACE_OID, matching the worker. Not an arbitrary choice — changing it
+# renames every episode.
+_UUID_NAMESPACE = uuid.NAMESPACE_OID
+
+_DATE_IN_TEXT_RE = re.compile(r"(\d{8})")
+# ISO-8601 dates are the common shape in session metadata (`recorded_at`), and
+# their hyphens defeat the 8-consecutive-digit match the worker uses on compact
+# S3 keys. Matched first and compacted, so a metadata timestamp and a path date
+# resolve to the same value instead of the 00000000 sentinel.
+_ISO_DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})")
+
+
+def normalize_device(raw: str) -> str:
+    """Lowercase, hyphens to underscores — the worker's normalization."""
+    return raw.lower().replace("-", "_")
+
+
+def derive_episode_id(date: str, device: str, session: str) -> str:
+    """The deterministic episode id the pipeline will assign.
+
+    Mirrors ``worker/ego/preprocess/concat.py::derive_episode_id``. Keep the
+    format string exactly as-is.
+    """
+    name = f"gilabs-ingest:{normalize_device(device)}:{session}:{date}"
+    return str(uuid.uuid5(_UUID_NAMESPACE, name))
+
+
+def extract_date(*values: str | None) -> str:
+    """First 8-digit run found across ``values``, else the worker's ``00000000``.
+
+    The worker scans the S3 key for a date; the CLI has better sources (session
+    metadata, then the session directory name), but falls back to the same
+    sentinel so an undated session lands on the same id either way.
+    """
+    for value in values:
+        if not value:
+            continue
+        iso = _ISO_DATE_RE.search(value)
+        if iso:
+            return f"{iso.group(1)}{iso.group(2)}{iso.group(3)}"
+        compact = _DATE_IN_TEXT_RE.search(value)
+        if compact:
+            return compact.group(1)
+    return "00000000"
+
+
+def episode_key(date: str, device: str, session: str) -> str:
+    """Path segment for this episode inside the batch prefix.
+
+    Human-readable on purpose: this is what a vendor sees in an error message
+    and what an engineer greps for in the bucket. Uniqueness within a batch is
+    enforced by the server at seal, not by this string.
+    """
+    return f"{date}/{normalize_device(device)}/{session}"