outo-models-cli 0.1.0__tar.gz → 0.2.0__tar.gz

{outo_models_cli-0.1.0 → outo_models_cli-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: outo-models-cli
-Version: 0.1.0
+Version: 0.2.0
 Summary: Command-line client for self-hosted outo-models servers — auth, repo, download, and upload.
 Project-URL: Homepage, https://github.com/outo-models/outo-models
 Project-URL: Documentation, https://github.com/outo-models/outo-models/blob/main/omc/README.md
@@ -47,7 +47,7 @@ Both `omc` and `outo-models-cli` console scripts are installed.
 
 ```bash
 # 1. Log in (prompts for a PAT, masked)
-omc auth login --server http://192.168.0.239
+omc auth login --server https://models.example.com
 
 # 2. Verify
 omc auth whoami
@@ -65,6 +65,12 @@ omc download alice/my-model --local-dir ./my-model
 omc upload alice/my-model ./checkpoints --path-in-repo weights --message "v1"
 ```
 
+Files larger than 100 MiB are uploaded through Git LFS automatically;
+the CLI partitions the input set, runs the LFS batch + PUT dance for
+each large file, and commits the resulting pointer text alongside any
+small files in a single commit. No `git lfs track` setup is required
+on the user side.
+
 ## Commands
 
 ```

{outo_models_cli-0.1.0 → outo_models_cli-0.2.0}/README.md

@@ -16,7 +16,7 @@ Both `omc` and `outo-models-cli` console scripts are installed.
 
 ```bash
 # 1. Log in (prompts for a PAT, masked)
-omc auth login --server http://192.168.0.239
+omc auth login --server https://models.example.com
 
 # 2. Verify
 omc auth whoami
@@ -34,6 +34,12 @@ omc download alice/my-model --local-dir ./my-model
 omc upload alice/my-model ./checkpoints --path-in-repo weights --message "v1"
 ```
 
+Files larger than 100 MiB are uploaded through Git LFS automatically;
+the CLI partitions the input set, runs the LFS batch + PUT dance for
+each large file, and commits the resulting pointer text alongside any
+small files in a single commit. No `git lfs track` setup is required
+on the user side.
+
 ## Commands
 
 ```

{outo_models_cli-0.1.0 → outo_models_cli-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "outo-models-cli"
-version = "0.1.0"
+version = "0.2.0"
 description = "Command-line client for self-hosted outo-models servers — auth, repo, download, and upload."
 readme = "README.md"
 requires-python = ">=3.12,<3.13"

{outo_models_cli-0.1.0 → outo_models_cli-0.2.0}/src/outo_models_cli/__init__.py

@@ -15,5 +15,5 @@ else. The server itself ships separately as `outo-models`.
 
 from __future__ import annotations
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 __all__ = ["__version__"]

{outo_models_cli-0.1.0 → outo_models_cli-0.2.0}/src/outo_models_cli/api/__init__.py

@@ -189,6 +189,22 @@ def display_filename(path_str: str, *, file_root: Path | None, idx: int) -> str:
 # sites can write `api.me(...)` instead of `api.auth.me(...)`.
 
 from outo_models_cli.api.auth import me  # noqa: E402
+from outo_models_cli.api.lfs import (  # noqa: E402
+    LFS_CONTENT_TYPE,
+    DedupMap,
+    LargeFile,
+    Partition,
+    dedupe_objects,
+    partition_files,
+    pointer_text,
+    pointers_for,
+)
+from outo_models_cli.api.lfs_batch import (  # noqa: E402
+    BatchAction,
+    BatchObjectError,
+    batch_upload,
+)
+from outo_models_cli.api.lfs_upload import upload_objects  # noqa: E402
 from outo_models_cli.api.repos import (  # noqa: E402
     create_repo,
     delete_repo,
@@ -201,23 +217,35 @@ from outo_models_cli.api.repos import (  # noqa: E402
 from outo_models_cli.api.upload import upload  # noqa: E402
 
 __all__ = [
+    "LFS_CONTENT_TYPE",
+    "BatchAction",
+    "BatchObjectError",
+    "DedupMap",
     "FileEntry",
+    "LargeFile",
+    "Partition",
     "RepoDetail",
     "RepoSummary",
     "UploadResult",
     "WhoAmI",
+    "batch_upload",
     "create_repo",
+    "dedupe_objects",
     "delete_repo",
     "display_filename",
     "get_repo",
     "list_files",
     "list_repos",
     "me",
+    "partition_files",
+    "pointer_text",
+    "pointers_for",
     "resolve_url",
     "send",
     "summary_from",
     "unwrap",
     "upload",
+    "upload_objects",
     "walk_repo",
     "with_client",
 ]

outo_models_cli-0.2.0/src/outo_models_cli/api/lfs.py

@@ -0,0 +1,229 @@
+"""Git LFS upload protocol — partition, batch, stream-PUT, pointer text.
+
+The CLI drives the four LFS endpoints the server implements:
+
+    POST  /{owner}/{name}.git/info/lfs/objects/batch
+    PUT   /{owner}/{name}.git/info/lfs/objects/{oid}
+    GET   /{owner}/{name}.git/info/lfs/objects/{oid}
+    GET   /{owner}/{name}/resolve/{revision}/{path:path}    # redirects to the above for LFS blobs
+
+The upload path runs at the boundary between two regimes:
+
+    * Files at or below the per-file multipart cap (100 MiB) ride the
+      existing `POST /api/repos/.../upload` endpoint unchanged.
+    * Files above the cap use the LFS batch + PUT dance, then are
+      committed to the repo as LFS pointer text. The commit happens via
+      the same multipart endpoint — the pointer bytes are tiny so they
+      fit the multipart limit with no special-casing.
+
+The download path is the mirror image: `GET /resolve/...` returns a 302
+to `/info/lfs/objects/{oid}` whenever the on-tree blob is an LFS
+pointer, and the CLI's Basic-auth client (see `http.build_basic_async_client`)
+follows that redirect transparently because httpx forwards the
+`Authorization` header on same-origin redirects.
+
+The wire format is the canonical git-lfs batch shape:
+
+    Request:  {"operation": "upload", "objects": [{"oid", "size"}, ...]}
+    Response: {"objects": [{"oid", "size", "actions": {"upload": {"href", "header", "expires_in"}}}
+                            | "error": {"code", "message"}]
+               | (no `actions` when the object is already on the server)}
+
+Auth is HTTP Basic (`username:PAT`) — Bearer is not accepted on the LFS
+surface, so this module assumes the client was built with
+`build_basic_client` / `build_basic_async_client`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+LFS_CONTENT_TYPE = "application/vnd.git-lfs+json"
+
+#: Chunk size used when streaming a file into the LFS PUT body. Big
+#: enough to keep the per-chunk syscalls off the hot path; small enough
+#: to bound peak memory at O(chunks-in-flight) regardless of file size.
+PUT_CHUNK_BYTES = 1024 * 1024
+
+#: Same chunk size used when computing the sha256 alongside the PUT
+#: stream. Identical to `PUT_CHUNK_BYTES` so a single read loop drives both.
+HASH_CHUNK_BYTES = 1024 * 1024
+
+
+# ---------------------------------------------------------------------------
+# Pointer text
+# ---------------------------------------------------------------------------
+
+
+def pointer_text(oid: str, size: int) -> bytes:
+    """Render the canonical git-lfs pointer text for `(oid, size)`.
+
+    The server's resolve endpoint sniffs the leading `version https://git-lfs`
+    line and redirects to the object URL; both the line order and the
+    trailing newline are part of the wire contract and MUST match.
+    """
+    return (f"version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n").encode()
+
+
+# ---------------------------------------------------------------------------
+# File partitioning + streaming sha256
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class LargeFile:
+    """A file that exceeds the multipart cap and must travel through LFS.
+
+    `oid` is computed by streaming the file once at partition time. The
+    CLI never loads the bytes into memory; both the digest and the
+    subsequent PUT body come from the same on-disk file via independent
+    read passes.
+    """
+
+    path: Path
+    oid: str
+    size: int
+
+
+@dataclass(frozen=True, slots=True)
+class Partition:
+    """Result of splitting a user-supplied file set into small / large buckets.
+
+    `small` keeps the original `Path` order so the on-disk filename
+    ordering round-trips into the multipart `files[]` list. `large` is
+    ordered by descending size so the biggest object (whose PUT takes
+    the longest) gets a tighter Rich progress ETA — UX nicety, not a
+    protocol requirement.
+    """
+
+    small: list[Path]
+    large: list[LargeFile] = field(default_factory=list)
+
+
+def _sha256_of(path: Path) -> tuple[str, int]:
+    """Stream `path` through sha256; return `(hex_digest, size)`.
+
+    A 1 MiB read buffer keeps memory bounded regardless of file size.
+    Errors from the underlying `open()` (file removed between the
+    partition step and the PUT) surface unchanged so the caller's
+    `OmcError` mapping sees the real cause.
+    """
+    hasher = hashlib.sha256()
+    total = 0
+    with path.open("rb") as fp:
+        while True:
+            chunk = fp.read(HASH_CHUNK_BYTES)
+            if not chunk:
+                break
+            hasher.update(chunk)
+            total += len(chunk)
+    return hasher.hexdigest(), total
+
+
+def partition_files(files: Iterable[Path], *, cap_bytes: int) -> Partition:
+    """Split `files` into small (multipart) and large (LFS) buckets.
+
+    The function streams each large file exactly once to compute its
+    sha256 — the same digest the server's LFS PUT handler will verify
+    at completion. Identical `(oid, size)` pairs are detected at batch
+    time, not here, so the caller still sees one entry per file path;
+    dedup happens against the wire-shape objects.
+    """
+    from outo_models_cli.errors import BadResponseError
+
+    small: list[Path] = []
+    large: list[LargeFile] = []
+    for f in files:
+        size = f.stat().st_size
+        if size <= cap_bytes:
+            small.append(f)
+            continue
+        oid, observed = _sha256_of(f)
+        if observed != size:
+            # Defensive: `stat().st_size` could race with concurrent
+            # writers. Surface as `BadResponseError`-shaped failure so
+            # the user sees a clean English line, not a Python traceback.
+            raise BadResponseError(
+                f"Size of {f} changed during hashing ({observed} != {size}); retry.",
+            )
+        large.append(LargeFile(path=f, oid=oid, size=size))
+    large.sort(key=lambda lf: lf.size, reverse=True)
+    return Partition(small=small, large=large)
+
+
+# ---------------------------------------------------------------------------
+# Deduplication — identical (oid, size) pairs collapse to one batch entry
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class DedupMap:
+    """Mapping from a deduped `(oid, size)` back to the files that need it.
+
+    The server's batch response carries one entry per `(oid, size)`. If
+    the user happens to have two different paths with identical content
+    (a common case in model repos that bundle the same checkpoint twice
+    under different names), only one PUT is required and the pointer
+    text is reused at commit time.
+    """
+
+    key: tuple[str, int]
+    paths: tuple[Path, ...]
+
+
+def dedupe_objects(items: list[LargeFile]) -> tuple[list[dict[str, Any]], list[DedupMap]]:
+    """Collapse duplicate `(oid, size)` items; return (batch_entries, path_maps).
+
+    `batch_entries` is the list of `{"oid", "size"}` dicts to send in
+    the LFS batch body — order matches the first occurrence in `items`
+    so the partition-time ordering survives. `path_maps` carries the
+    full set of paths that need each pointer text at commit time.
+    """
+    seen: dict[tuple[str, int], int] = {}
+    entries: list[dict[str, Any]] = []
+    path_lists: list[list[Path]] = []
+
+    for item in items:
+        key = (item.oid, item.size)
+        idx = seen.get(key)
+        if idx is None:
+            seen[key] = len(entries)
+            entries.append({"oid": item.oid, "size": item.size})
+            path_lists.append([item.path])
+        else:
+            path_lists[idx].append(item.path)
+
+    maps = [DedupMap(key=key, paths=tuple(path_lists[idx])) for key, idx in seen.items()]
+    return entries, maps
+
+
+def pointers_for(large: list[LargeFile]) -> dict[str, bytes]:
+    """Return a `{oid: pointer_text}` mapping for every deduped object.
+
+    The mapping is keyed on oid only because dedupe means a single
+    pointer text covers every file that hashes to that oid; the caller
+    expands it back to one entry per path when building the multipart
+    request.
+    """
+    out: dict[str, bytes] = {}
+    for item in large:
+        out.setdefault(item.oid, pointer_text(item.oid, item.size))
+    return out
+
+
+__all__ = [
+    "HASH_CHUNK_BYTES",
+    "LFS_CONTENT_TYPE",
+    "PUT_CHUNK_BYTES",
+    "DedupMap",
+    "LargeFile",
+    "Partition",
+    "dedupe_objects",
+    "partition_files",
+    "pointer_text",
+    "pointers_for",
+]

outo_models_cli-0.2.0/src/outo_models_cli/api/lfs_batch.py

@@ -0,0 +1,162 @@
+"""LFS batch request/response parsing.
+
+The server's `/info/lfs/objects/batch` endpoint accepts an
+`{"operation": "upload", "objects": [{"oid", "size"}]}` body and returns
+a response with one entry per object. Each entry has either an
+`actions.upload` block (with an absolute `href` the client streams the
+raw bytes to) or a per-object `error` block (413 size cap, 413 quota,
+404 unknown object for the download operation). An object that is
+already on the server comes back WITHOUT `actions` AND without `error`
+— the client interprets that as "skip the PUT, write pointer text".
+
+This module is the pure wire-format layer: it sends the request and
+parses the response into typed dataclasses that the upload command can
+drain with one loop. The actual streaming PUT lives in `lfs_upload.py`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import httpx
+
+from outo_models_cli.api import send, unwrap
+from outo_models_cli.api.lfs import LFS_CONTENT_TYPE
+from outo_models_cli.errors import BadResponseError, map_response_error
+
+
+@dataclass(frozen=True, slots=True)
+class BatchAction:
+    """Wire-level action handed back by the server for one object."""
+
+    oid: str
+    size: int
+    href: str
+    header: dict[str, str] = field(default_factory=dict)
+    expires_in: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class BatchObjectError:
+    """Per-object failure surfaced by the LFS batch response."""
+
+    oid: str
+    size: int
+    code: int
+    message: str
+
+
+def batch_upload(
+    client: httpx.Client,
+    *,
+    owner: str,
+    name: str,
+    objects: list[dict[str, object]],
+) -> tuple[list[BatchAction], list[BatchObjectError], list[str]]:
+    """POST the LFS upload batch; split the response into actions / errors / present.
+
+    Returns:
+        * `actions` — objects the server wants uploaded, in input order.
+        * `errors` — per-object failures (413 size cap, 413 quota, ...).
+        * `present` — oids the server already has; the caller skips the PUT
+          and still writes the pointer text for these at commit time.
+    """
+    body = {
+        "operation": "upload",
+        "transfers": ["basic"],
+        "objects": objects,
+    }
+    response = send(
+        client,
+        "POST",
+        f"/{owner}/{name}.git/info/lfs/objects/batch",
+        headers={"Accept": LFS_CONTENT_TYPE, "Content-Type": LFS_CONTENT_TYPE},
+        json=body,
+    )
+    if response.status_code == 404:
+        raise BadResponseError(
+            "Server does not support Git LFS at this endpoint. "
+            "Upgrade the server or contact the operator.",
+        )
+    if response.status_code == 406:
+        raise BadResponseError(
+            "Server rejected the LFS request (missing Accept: application/vnd.git-lfs+json).",
+        )
+    if response.status_code == 415:
+        raise BadResponseError(
+            "Server rejected the LFS request (wrong Content-Type).",
+        )
+    if response.status_code >= 400:
+        raise map_response_error(response)
+    payload = unwrap(response)
+    raw_objects = payload.get("objects")
+    if not isinstance(raw_objects, list):
+        raise BadResponseError("LFS batch response is missing `objects`.")
+
+    actions: list[BatchAction] = []
+    errors: list[BatchObjectError] = []
+    present: list[str] = []
+
+    for entry in raw_objects:
+        if not isinstance(entry, dict):
+            continue
+        oid = str(entry.get("oid", ""))
+        size_val = entry.get("size", 0)
+        size = int(size_val) if isinstance(size_val, (int, float)) else 0
+        error_raw = entry.get("error")
+        if isinstance(error_raw, dict):
+            code_raw = error_raw.get("code")
+            code = int(code_raw) if isinstance(code_raw, (int, float)) else 0
+            message = str(error_raw.get("message", ""))
+            errors.append(BatchObjectError(oid=oid, size=size, code=code, message=message))
+            continue
+        actions_raw = entry.get("actions")
+        if not isinstance(actions_raw, dict):
+            # No actions AND no error → object is already stored server-side.
+            present.append(oid)
+            continue
+        upload_raw = actions_raw.get("upload")
+        if not isinstance(upload_raw, dict):
+            errors.append(
+                BatchObjectError(
+                    oid=oid,
+                    size=size,
+                    code=500,
+                    message="LFS response missing `actions.upload`.",
+                )
+            )
+            continue
+        href = str(upload_raw.get("href", ""))
+        if not href:
+            errors.append(
+                BatchObjectError(
+                    oid=oid,
+                    size=size,
+                    code=500,
+                    message="LFS response missing upload `href`.",
+                )
+            )
+            continue
+        header_raw = upload_raw.get("header") or {}
+        header: dict[str, str] = {}
+        if isinstance(header_raw, dict):
+            for k, v in header_raw.items():
+                if isinstance(k, str) and isinstance(v, str):
+                    header[k] = v
+        expires_in_raw = upload_raw.get("expires_in")
+        expires_in: int | None = None
+        if isinstance(expires_in_raw, (int, float)):
+            expires_in = int(expires_in_raw)
+        actions.append(
+            BatchAction(
+                oid=oid,
+                size=size,
+                href=href,
+                header=header,
+                expires_in=expires_in,
+            )
+        )
+    return actions, errors, present
+
+
+__all__ = ["BatchAction", "BatchObjectError", "batch_upload"]