outo-models-cli 0.1.0__py3-none-any.whl

outo_models_cli/downloader/__init__.py

@@ -0,0 +1,161 @@
+"""Recursive, resumable downloader.
+
+Why one package owns the whole flow (not the CLI command):
+
+    * The CLI command renders progress bars; the downloader is the only
+      layer that knows chunk sizes, byte counts, and resume semantics.
+    * A single transport (`httpx.AsyncClient`) avoids the concurrency
+      trap of running sync calls inside a thread pool: one connection
+      pool, one auth header, one close path.
+    * Streaming integrity checks (ETag / Content-Length) belong with
+      the byte-level logic; mixing them into the CLI renderer would
+      scatter the wire contract across files.
+
+The public entry point is `outo_models_cli.downloader.download_repo` —
+a coroutine that walks the remote tree, schedules parallel file
+transfers under an asyncio semaphore, and yields per-file outcomes the
+CLI renderer can attach Rich bars to.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import httpx
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    TextColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
+
+from outo_models_cli import api
+from outo_models_cli.downloader.transfer import FileOutcome, transfer_one
+from outo_models_cli.downloader.walk import walk_repo_async
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadConfig:
+    """All knobs that change the downloader behavior.
+
+    Centralising them in a frozen dataclass keeps the download coroutine
+    signature short (the function takes one config object) and lets
+    tests pin every value at once. Defaults match the values the CLI
+    commands set when invoked from `omc download`.
+    """
+
+    revision: str = "main"
+    include: list[str] = field(default_factory=list)
+    exclude: list[str] = field(default_factory=list)
+    local_dir: Path = Path(".")
+    max_workers: int = 8
+
+
+def build_progress() -> Progress:
+    """Return the per-file Rich progress bar group.
+
+    Exposed for the CLI command so the same columns render in tests as
+    in production. A custom column set (instead of the default
+    `Progress()`) gives us a fixed-width `description` cell, which
+    keeps the multi-file bars aligned when filenames vary in length.
+    """
+    return Progress(
+        TextColumn("[bold blue]{task.description}[/bold blue]", justify="left"),
+        BarColumn(bar_width=40),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        TextColumn("eta"),
+        TimeRemainingColumn(),
+        expand=True,
+    )
+
+
+ProgressCallback = Callable[[Progress, Any], None]
+"""Hook the CLI uses to bind a per-file task. Kept as a thin callback
+so the downloader never imports the CLI renderer directly."""
+
+
+async def download_repo(
+    *,
+    client: httpx.AsyncClient,
+    base_url: str,
+    token: str,
+    owner: str,
+    name: str,
+    config: DownloadConfig,
+    on_progress: Callable[[str], None] | None = None,
+) -> list[FileOutcome]:
+    """Walk `owner/name` and stream every file under `config.local_dir`.
+
+    Returns one `FileOutcome` per file, in the order the tree walk
+    produced (depth-first, alphabetical inside each directory). The
+    caller is expected to run this coroutine under
+    `asyncio.run(...)` and is free to wrap it with their own progress
+    rendering — `on_progress` is a per-file completion notification the
+    CLI uses to print a one-line summary without subscribing to the
+    Rich bar stream.
+    """
+    files = await walk_repo_async(
+        client,
+        owner=owner,
+        name=name,
+        revision=config.revision,
+        include=config.include,
+        exclude=config.exclude,
+    )
+
+    config.local_dir.mkdir(parents=True, exist_ok=True)
+    semaphore = asyncio.Semaphore(max(1, config.max_workers))
+
+    progress = build_progress()
+    outcomes: list[FileOutcome] = []
+    with progress:
+        tasks: list[asyncio.Future[FileOutcome]] = []
+
+        async def _bound(entry: api.FileEntry) -> FileOutcome:
+            target = config.local_dir / entry.path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            task_id = progress.add_task(
+                entry.path,
+                total=entry.size_bytes,
+                completed=0,
+            )
+            async with semaphore:
+                outcome = await transfer_one(
+                    client,
+                    owner=owner,
+                    name=name,
+                    revision=config.revision,
+                    file=entry,
+                    target=target,
+                    progress=progress,
+                    task_id=task_id,
+                )
+            progress.remove_task(task_id)
+            return outcome
+
+        for entry in files:
+            tasks.append(asyncio.ensure_future(_bound(entry)))
+
+        for coro in asyncio.as_completed(tasks):
+            outcome = await coro
+            outcomes.append(outcome)
+            if on_progress is not None:
+                on_progress(outcome.path)
+
+    return outcomes
+
+
+__all__ = [
+    "DownloadConfig",
+    "FileOutcome",
+    "ProgressCallback",
+    "build_progress",
+    "download_repo",
+]

outo_models_cli/downloader/transfer.py

@@ -0,0 +1,140 @@
+"""Stream one file with resume + integrity semantics."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import httpx
+from rich.progress import Progress
+
+from outo_models_cli import api
+from outo_models_cli.errors import NotFoundError, map_response_error, map_transport_error
+
+# 1 MiB chunk size — small enough to keep memory bounded even when the
+# CLI streams many parallel downloads; large enough that the chunk
+# overhead is negligible against the per-byte disk cost.
+CHUNK_BYTES = 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class FileOutcome:
+    """One row in the download report the CLI prints at the end."""
+
+    path: str
+    bytes_downloaded: int
+    resumed: bool
+    verified: bool  # True when an ETag / sha check was performed
+
+
+async def transfer_one(
+    client: httpx.AsyncClient,
+    *,
+    owner: str,
+    name: str,
+    revision: str,
+    file: api.FileEntry,
+    target: Path,
+    progress: Progress,
+    task_id: Any,
+) -> FileOutcome:
+    """Stream `file` from the server to `target`, honouring `.part` resume."""
+    url = api.resolve_url(owner=owner, name=name, revision=revision, path=file.path)
+    part_path = target.with_suffix(target.suffix + ".part")
+    existing_size = part_path.stat().st_size if part_path.exists() else 0
+
+    headers: dict[str, str] = {}
+    if existing_size > 0:
+        headers["Range"] = f"bytes={existing_size}-"
+
+    try:
+        async with client.stream("GET", url, headers=headers) as response:
+            if response.status_code == 416:
+                # Range not satisfiable → server's file shrunk or our
+                # .part is corrupt. Restart from scratch.
+                part_path.unlink(missing_ok=True)
+                return await transfer_one(
+                    client,
+                    owner=owner,
+                    name=name,
+                    revision=revision,
+                    file=file,
+                    target=target,
+                    progress=progress,
+                    task_id=task_id,
+                )
+            if response.status_code == 404:
+                raise NotFoundError(f"File not found on server: {file.path}")
+            if response.status_code >= 400:
+                raise map_response_error(response)
+
+            server_etag = response.headers.get("ETag")
+            content_length_raw = response.headers.get("Content-Length")
+            content_length = (
+                int(content_length_raw)
+                if content_length_raw and content_length_raw.isdigit()
+                else None
+            )
+            expected_total = file.size_bytes
+
+            if response.status_code == 200:
+                # Server ignored Range and is sending the full file
+                # (ETag mismatch, or no Range header was sent).
+                resumed = False
+                part_path.unlink(missing_ok=True)
+                mode = "wb"
+                existing_size = 0
+                download_total = expected_total if expected_total is not None else 0
+            else:  # 206 Partial Content
+                resumed = existing_size > 0
+                mode = "ab"
+                download_total = (
+                    (expected_total - existing_size)
+                    if expected_total is not None and existing_size <= expected_total
+                    else 0
+                )
+
+            bytes_written = existing_size
+            progress.update(
+                task_id,
+                total=(download_total + bytes_written) or None,
+                completed=bytes_written,
+            )
+
+            # ETag may be a quoted hex sha or a weak validator. The
+            # integrity check is a verbatim match against the value the
+            # server sent on this response; mismatches between calls
+            # signal "the file changed; restart from scratch" above.
+            sha = hashlib.sha256() if server_etag and "sha256" in server_etag.lower() else None
+
+            with part_path.open(mode) as fp:
+                async for chunk in response.aiter_bytes(chunk_size=CHUNK_BYTES):
+                    if not chunk:
+                        continue
+                    fp.write(chunk)
+                    bytes_written += len(chunk)
+                    progress.update(task_id, completed=bytes_written)
+                    if sha is not None:
+                        sha.update(chunk)
+
+        # Atomic rename to the final target. If anything in this block
+        # raises (disk full, permissions), the .part file is left on
+        # disk for the next run to resume.
+        os.replace(part_path, target)
+
+        verified = server_etag is not None or content_length is not None
+
+        return FileOutcome(
+            path=file.path,
+            bytes_downloaded=bytes_written,
+            resumed=resumed,
+            verified=verified,
+        )
+    except httpx.HTTPError as exc:
+        raise map_transport_error(exc) from exc
+
+
+__all__ = ["transfer_one"]

outo_models_cli/downloader/walk.py

@@ -0,0 +1,89 @@
+"""Recursive tree walk over the server's `/files` endpoint."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from outo_models_cli import api
+from outo_models_cli.errors import OmcError, map_response_error, map_transport_error
+from outo_models_cli.matchers import passes
+
+
+async def list_files_async(
+    client: httpx.AsyncClient,
+    *,
+    owner: str,
+    name: str,
+    path: str,
+    revision: str,
+) -> list[api.FileEntry]:
+    """Async equivalent of `api.list_files` (uses the shared async client)."""
+    try:
+        response = await client.get(
+            f"/api/repos/{owner}/{name}/files",
+            params={
+                "path": path,
+                **({"revision": revision} if revision else {}),
+            },
+        )
+    except httpx.HTTPError as exc:
+        raise map_transport_error(exc) from exc
+    if response.status_code >= 400:
+        raise map_response_error(response)
+    payload: Any = response.json()
+    rows = payload.get("entries", []) if isinstance(payload, dict) else []
+    if not isinstance(rows, list):
+        raise OmcError("`/files` returned a malformed `entries` field.")
+    entries: list[api.FileEntry] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        size = row.get("size_bytes")
+        entries.append(
+            api.FileEntry(
+                name=str(row.get("name", "")),
+                path=str(row.get("path", "")),
+                kind=str(row.get("kind", "")),
+                size_bytes=int(size) if isinstance(size, (int, float)) else None,
+            )
+        )
+    return entries
+
+
+async def walk_repo_async(
+    client: httpx.AsyncClient,
+    *,
+    owner: str,
+    name: str,
+    revision: str,
+    include: list[str],
+    exclude: list[str],
+) -> list[api.FileEntry]:
+    """Recursively enumerate every file under a repo (depth-first)."""
+    out: list[api.FileEntry] = []
+
+    async def visit(directory: str) -> None:
+        rows = await list_files_async(
+            client,
+            owner=owner,
+            name=name,
+            path=directory,
+            revision=revision,
+        )
+        for entry in rows:
+            if entry.kind == "dir":
+                await visit(entry.path)
+            elif entry.kind == "file" and passes(
+                entry.path,
+                include=include,
+                exclude=exclude,
+            ):
+                out.append(entry)
+
+    await visit("")
+    return out
+
+
+__all__ = ["list_files_async", "walk_repo_async"]

outo_models_cli/errors.py

@@ -0,0 +1,179 @@
+"""Typed error hierarchy for `outo-models-cli`.
+
+Every failure surfaced to the user flows through an `OmcError` subclass so
+the CLI can render a single clean English line and exit 1, never a Python
+traceback (matches the server's `OutoError` discipline in AGENTS.md §2.1).
+
+The `code` is a stable machine identifier the caller (a shell wrapper, a
+test) can branch on without parsing prose. The `message` is human-facing
+English and may evolve.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+
+class OmcError(Exception):
+    """Base class for every CLI-facing error.
+
+    `code` is the stable identifier (no spaces, no punctuation). `message`
+    is the user-facing English line printed by `render_error()`.
+    """
+
+    code: str = "omc_error"
+
+    def __init__(self, message: str = "", *, code: str | None = None) -> None:
+        super().__init__(message)
+        if code is not None:
+            self.code = code
+
+
+class ConfigError(OmcError):
+    """Configuration is missing, malformed, or refuses to load."""
+
+    code = "config_error"
+
+
+class AuthRequiredError(OmcError):
+    """No credentials are stored for the requested server."""
+
+    code = "auth_required"
+
+
+class AuthInvalidError(OmcError):
+    """The server rejected the supplied token (HTTP 401)."""
+
+    code = "auth_invalid"
+
+
+class ForbiddenError(OmcError):
+    """The server rejected the request (HTTP 403)."""
+
+    code = "forbidden"
+
+
+class NotFoundError(OmcError):
+    """The server replied 404 for the requested resource."""
+
+    code = "not_found"
+
+
+class FileTooLargeError(OmcError):
+    """Single file exceeds the 100 MiB multipart cap; use git + LFS."""
+
+    code = "file_too_large"
+
+
+class ValidationFailedError(OmcError):
+    """The server rejected the request as malformed (HTTP 422)."""
+
+    code = "validation_failed"
+
+
+class ServerUnreachableError(OmcError):
+    """Cannot reach the server (DNS, connection refused, timeout)."""
+
+    code = "server_unreachable"
+
+
+class BadResponseError(OmcError):
+    """The server replied but the response could not be consumed."""
+
+    code = "bad_response"
+
+
+class FileMissingError(OmcError):
+    """A local file passed to `omc upload` does not exist."""
+
+    code = "file_missing"
+
+
+# Status code → exception class mapping. Anything outside this table is
+# treated as a generic `BadResponseError`.
+_STATUS_MAP: dict[int, type[OmcError]] = {
+    401: AuthInvalidError,
+    403: ForbiddenError,
+    404: NotFoundError,
+    413: FileTooLargeError,
+    422: ValidationFailedError,
+}
+
+
+def map_response_error(response: httpx.Response, *, fallback: str | None = None) -> OmcError:
+    """Convert an `httpx.Response` into the appropriate `OmcError`.
+
+    Args:
+        response: The error response (status >= 400).
+        fallback: Override for the rendered message; defaults to the
+            status-mapped text below.
+
+    The function inspects `response.status_code` first and falls back to
+    `BadResponseError` for everything else, so a future 418 from the
+    server still surfaces as a clean line instead of leaking a traceback.
+    """
+    status = response.status_code
+    detail: str = ""
+    try:
+        payload: Any = response.json()
+    except ValueError:
+        payload = None
+    if isinstance(payload, dict):
+        raw = payload.get("detail") or payload.get("message") or payload.get("error")
+        if isinstance(raw, str):
+            detail = raw
+    message = fallback or _default_message(status, detail)
+    cls = _STATUS_MAP.get(status, BadResponseError)
+    return cls(message, code=cls.code)
+
+
+def map_transport_error(exc: httpx.HTTPError) -> OmcError:
+    """Wrap an `httpx.HTTPError` as `ServerUnreachableError`.
+
+    Used by the API client so transport-layer failures (DNS, refusal,
+    timeouts, TLS) surface with a single stable message instead of leaking
+    the underlying class name into the operator's terminal.
+    """
+    return ServerUnreachableError(
+        "Cannot reach the server. Check the URL, network, and that the server is running.",
+    )
+
+
+def _default_message(status: int, detail: str) -> str:
+    if detail:
+        return detail
+    if status == 401:
+        return "Not authenticated. Run `omc auth login --server <url>` to store a token."
+    if status == 403:
+        return "Forbidden: you do not have permission for this operation."
+    if status == 404:
+        return "Resource not found."
+    if status == 413:
+        return (
+            "Upload too large. Files over 100 MiB must be uploaded via git + LFS. "
+            "Try splitting the file or using `git lfs track`."
+        )
+    if status == 422:
+        return "The server rejected the request as invalid. Check paths and arguments."
+    if 400 <= status < 500:
+        return f"Request failed (HTTP {status})."
+    return f"Server returned an unexpected error (HTTP {status})."
+
+
+__all__ = [
+    "AuthInvalidError",
+    "AuthRequiredError",
+    "BadResponseError",
+    "ConfigError",
+    "FileMissingError",
+    "FileTooLargeError",
+    "ForbiddenError",
+    "NotFoundError",
+    "OmcError",
+    "ServerUnreachableError",
+    "ValidationFailedError",
+    "map_response_error",
+    "map_transport_error",
+]

outo_models_cli/http.py

@@ -0,0 +1,73 @@
+"""HTTP client construction helpers.
+
+Centralises the timeout / header / base-URL defaults so every API call
+shares the same transport. The factory returns a fresh `httpx.Client`
+each call — short-lived, single-purpose clients are easier to reason
+about than a long-lived shared one (no state bleed between commands),
+and the cost is one TCP handshake per CLI invocation.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+# 30s connect / read budget — the server is local in a self-hosted
+# deployment, but a slow disk or large multipart may legitimately hold
+# the connection for a few seconds. Anything longer is genuinely hung.
+_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
+
+
+def build_client(
+    base_url: str,
+    token: str,
+    *,
+    follow_redirects: bool = False,
+    **client_kwargs: Any,
+) -> httpx.Client:
+    """Return an `httpx.Client` pre-wired with the bearer credential.
+
+    Args:
+        base_url: The full origin the server listens on. The path prefix
+            (`/api`, `/{owner}/{name}/resolve/...`) is appended per call.
+        token: Personal Access Token used as the bearer credential.
+        follow_redirects: Set to `True` for LFS pointer redirects. Other
+            endpoints MUST stay non-redirecting so a misconfigured proxy
+            cannot silently absorb a token via 30x.
+        **client_kwargs: Forwarded to `httpx.Client` (used by tests to
+            inject a `respx` transport via `transport=`).
+    """
+    headers = {"Authorization": f"Bearer {token}"}
+    return httpx.Client(
+        base_url=base_url.rstrip("/"),
+        timeout=_TIMEOUT,
+        headers=headers,
+        follow_redirects=follow_redirects,
+        **client_kwargs,
+    )
+
+
+def build_async_client(
+    base_url: str,
+    token: str,
+    *,
+    follow_redirects: bool = False,
+    **client_kwargs: Any,
+) -> httpx.AsyncClient:
+    """Async equivalent of `build_client`.
+
+    Used by the parallel `download` command. `base_url` is normalized the
+    same way as the sync version so URL joins are consistent.
+    """
+    headers = {"Authorization": f"Bearer {token}"}
+    return httpx.AsyncClient(
+        base_url=base_url.rstrip("/"),
+        timeout=_TIMEOUT,
+        headers=headers,
+        follow_redirects=follow_redirects,
+        **client_kwargs,
+    )
+
+
+__all__ = ["build_async_client", "build_client"]