withcache 0.8.10__tar.gz → 0.9.1__tar.gz

{withcache-0.8.10 → withcache-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: withcache
-Version: 0.8.10
+Version: 0.9.1
 Summary: Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)
 Project-URL: Homepage, https://github.com/safl/withcache
 Author-email: "Simon A. F. Lund" <safl@safl.dk>
@@ -14,6 +14,11 @@ Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: System :: Archiving :: Mirroring
 Requires-Python: >=3.11
+Requires-Dist: fastapi>=0.115
+Requires-Dist: itsdangerous>=2.1
+Requires-Dist: jinja2>=3.1
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: uvicorn[standard]>=0.30
 Description-Content-Type: text/markdown
 
 # withcache

{withcache-0.8.10 → withcache-0.9.1}/pyproject.toml

@@ -19,7 +19,30 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Topic :: System :: Archiving :: Mirroring",
 ]
-dependencies = []  # stdlib only, by design
+dependencies = [
+    # v0.9.0 shipped the FastAPI port: the daemon UI + control plane
+    # moved off stdlib http.server onto the same stack bty-web and
+    # nbdmux use. Keeps the trio's testing shape uniform (FastAPI
+    # TestClient across all three) and makes the shared chrome +
+    # settings-store patterns extractable into a trio-common lib.
+    # ``withcache.client`` / ``withcache.oras`` stay stdlib-only so
+    # downstream consumers (bty) don't inherit the framework floor.
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.30",
+    "jinja2>=3.1",
+    # Form-encoded POST parsing behind ``Form()`` deps.
+    "python-multipart>=0.0.9",
+    # Session cookie signing (SessionMiddleware).
+    "itsdangerous>=2.1",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8",
+    "ruff>=0.6",
+    # FastAPI TestClient uses httpx as its transport.
+    "httpx>=0.27",
+]
 
 [project.urls]
 Homepage = "https://github.com/safl/withcache"

{withcache-0.8.10 → withcache-0.9.1}/shim/build.zig.zon

@@ -2,7 +2,7 @@
     .name = .withcache_shim,
     // Zig requires a literal here; keep it in lockstep with the project's
     // single source (src/withcache/__init__.py) via `make bump` / `make version-check`.
-    .version = "0.8.10",
+    .version = "0.9.1",
     .fingerprint = 0xd7d96c5ed212ccaa,
     .minimum_zig_version = "0.16.0",
     .paths = .{

{withcache-0.8.10 → withcache-0.9.1}/src/withcache/__init__.py

@@ -1,4 +1,4 @@
-"""withcache — operator-curated, URL-keyed artifact cache for a small lab.
+"""withcache, operator-curated URL-keyed artifact cache for a small lab.
 
 - ``withcache-server`` (withcache.server:main): the cache-host.
 - ``curlwithcache`` / ``wgetwithcache``: transparent curl/wget shims, shipped
@@ -11,12 +11,16 @@
   uses it on a cold miss; library consumers (e.g. ``bty``) import it to
   validate catalog entries and pre-resolve content digests.
 
-All modules are stdlib-only and self-contained.
+Since v0.9.0 the daemon runs on FastAPI + Jinja + Bootstrap 5 + htmx
+(matching bty-web + nbdmux, the eventual ``trio-common`` extraction
+target); ``withcache.client``, ``withcache.oras``, and the shim
+launchers stay stdlib-only so downstream consumers (bty) don't
+inherit the framework floor.
 """
 
 from . import oras
 from .client import blob_url, cache_base, is_cached, serve_url
 
-__version__ = "0.8.10"
+__version__ = "0.9.1"
 
 __all__ = ["__version__", "blob_url", "cache_base", "is_cached", "oras", "serve_url"]

withcache-0.9.1/src/withcache/_api.py

@@ -0,0 +1,330 @@
+"""Byte-serving routes for the withcache FastAPI app.
+
+Ports ``GET /blob?url=<origin>`` and ``GET /b/<b64>/<name>`` (+
+their HEAD counterparts) from the stdlib ``server.py`` handler.
+This is the bty-critical surface: bty at
+``bty.web._withcache.blob_url`` builds these URLs and the live
+env hands them to a ``wget|dd`` chain, so the wire contract must
+stay byte-identical:
+
+- URL shape: ``/b/<urlsafe_b64(src_url)>/<name>``. Name segment is
+  decorative; the b64 token is the source of truth.
+- Cache hit -> 200 + ``Content-Type`` + ``Content-Length`` +
+  ``X-Withcache-Sha256`` + streamed body (64 KiB chunks).
+- Cache miss -> 404 with ``cache miss (recorded)`` body,
+  side-effect of enqueueing a background fetch when
+  ``auto_fetch`` is on and the store has capacity.
+- HEAD support: same headers, no body (probes from
+  bty.web._withcache.is_cached use HEAD).
+- ``Authorization`` request header forwarded onto the background
+  fetch worker so a token-gated origin (typical use case: an OCI
+  bearer minted by bty at catalog import time) can be pulled by
+  the worker with the same credential.
+
+Streaming shape: FastAPI ``StreamingResponse`` with a generator
+that reads chunks from the on-disk blob at 64 KiB reads;
+:class:`StreamRegistry` ticks progress per chunk so the operator
+Downloads page shows in-flight transfers as they run.
+"""
+
+from __future__ import annotations
+
+import base64
+import urllib.parse
+from collections.abc import Iterator
+from typing import Any
+
+from fastapi import Depends, FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse, PlainTextResponse, Response, StreamingResponse
+
+from .server import CHUNK, CatalogState, StreamRegistry, _oras_tag_moved, _serialise_catalog
+
+
+def _decode_blob_origin(path: str, query: str) -> str:
+    """Extract the source URL from either shape.
+
+    ``/blob?url=<origin>``: read ``url`` query param.
+    ``/b/<b64>/<name>``: base64-decode the first path segment. The
+    trailing ``<name>`` is decorative -- a client that derives
+    format from URL filename (e.g. bty's flash chain) gets the
+    right extension without the b64 token having to include it.
+    """
+    if path.startswith("/b/"):
+        token = path[len("/b/") :].split("/", 1)[0]
+        try:
+            return base64.urlsafe_b64decode(token + "=" * (-len(token) % 4)).decode("utf-8")
+        except (ValueError, UnicodeDecodeError):
+            return ""
+    return (urllib.parse.parse_qs(query).get("url") or [""])[0]
+
+
+def _serve_blob(
+    request: Request,
+    url: str,
+    head_only: bool,
+) -> Response:
+    """Shared implementation for GET and HEAD blob requests.
+
+    Returns 400 when the origin URL couldn't be decoded, 404 (with
+    miss recording + optional fetch enqueue) on cache miss, or 200
+    + a StreamingResponse on hit.
+    """
+    if not url:
+        return PlainTextResponse("missing url\n", status_code=400)
+
+    store = request.app.state.store
+    mgr = request.app.state.mgr
+    auto_fetch: bool = request.app.state.auto_fetch
+    streams: StreamRegistry = request.app.state.streams
+
+    row = store.get_blob(url)
+    if row is not None and _oras_tag_moved(url, row["sha256"]):
+        # Tag re-pushed since we cached it: drop the stale bytes so
+        # the miss branch below re-fetches the current content.
+        store.delete_blob(row["key"])
+        row = None
+
+    if row is None:
+        store.record_miss(url)
+        if auto_fetch and store.has_capacity():
+            # Forward the client's Authorization header onto the
+            # worker so a token-gated origin (fresh OCI bearer from
+            # bty at catalog import time) can be fetched. Narrow
+            # allowlist: Authorization only; /admin/fetch carries
+            # its own ``headers=`` payload for the curated path.
+            fwd_headers: dict[str, str] | None = None
+            auth_header = request.headers.get("Authorization")
+            if auth_header:
+                fwd_headers = {"Authorization": auth_header}
+            mgr.enqueue(url, headers=fwd_headers)
+        return PlainTextResponse("cache miss (recorded)\n", status_code=404)
+
+    path = store.blob_path(row["key"])
+    headers = {
+        "Content-Length": str(row["size"]),
+        "X-Withcache-Sha256": row["sha256"],
+    }
+    media_type = row["content_type"] or "application/octet-stream"
+
+    if head_only:
+        # No body; the shim's HEAD probe doesn't count as a served
+        # download so record_hit + stream registration are skipped.
+        return Response(status_code=200, media_type=media_type, headers=headers)
+
+    store.record_hit(row["key"])
+    client = f"{request.client.host}:{request.client.port}" if request.client else "unknown"
+    stream = streams.start(url=url, client=client, total=row["size"])
+
+    def _chunks() -> Iterator[bytes]:
+        """Read the blob in 64 KiB chunks. StreamingResponse iterates
+        this + writes each chunk to the client. Progress ticks fire
+        every :data:`StreamRegistry.PROGRESS_STRIDE` chunks so the
+        dashboard's 1 Hz refresh sees updates without lock-contention
+        on a busy box."""
+        sent = 0
+        ticks = 0
+        try:
+            with open(path, "rb") as f:
+                while True:
+                    chunk = f.read(CHUNK)
+                    if not chunk:
+                        break
+                    yield chunk
+                    sent += len(chunk)
+                    ticks += 1
+                    if ticks % StreamRegistry.PROGRESS_STRIDE == 0:
+                        streams.bump(stream.id, sent)
+                streams.bump(stream.id, sent)
+        finally:
+            streams.finish(stream.id)
+
+    return StreamingResponse(_chunks(), media_type=media_type, headers=headers)
+
+
+def register_api_routes(app: FastAPI) -> None:
+    """Attach the byte-serving routes to ``app``.
+
+    Runtime objects (``store``, ``mgr``, ``streams``, ``auto_fetch``)
+    are read from ``app.state`` at request time so tests + the
+    lifespan hook can inject fresh instances without rebuilding
+    the app.
+    """
+
+    @app.get("/blob", response_class=Response)
+    def blob_get_legacy(request: Request) -> Response:
+        """Legacy ``?url=<origin>`` shape. Kept for downstream
+        consumers that pinned this before the ``/b/<b64>/<name>``
+        canonical URL landed."""
+        url = _decode_blob_origin("/blob", request.url.query)
+        return _serve_blob(request, url, head_only=False)
+
+    @app.head("/blob")
+    def blob_head_legacy(request: Request) -> Response:
+        url = _decode_blob_origin("/blob", request.url.query)
+        return _serve_blob(request, url, head_only=True)
+
+    @app.get("/b/{token}/{name:path}", response_class=Response)
+    def blob_get_b64(token: str, name: str, request: Request) -> Response:
+        """Canonical ``/b/<b64>/<name>`` shape. bty's flash chain
+        builds these URLs; the ``<name>`` segment is decorative so
+        the client derives extension for downstream format probes."""
+        del name  # decorative segment; source of truth is the b64 token
+        url = _decode_blob_origin(f"/b/{token}/x", "")
+        return _serve_blob(request, url, head_only=False)
+
+    @app.head("/b/{token}/{name:path}")
+    def blob_head_b64(token: str, name: str, request: Request) -> Response:
+        del name
+        url = _decode_blob_origin(f"/b/{token}/x", "")
+        return _serve_blob(request, url, head_only=True)
+
+    # ---------- Catalog JSON API -----------------------------------------
+    #
+    # Bty consumes the catalog through these endpoints: withcache is the
+    # single source of truth for what images exist. Read is open (mirrors
+    # /blob's LAN-only trust model); writes gate on the same Bearer
+    # pattern nbdmux uses -- an ``Authorization: Bearer <pw>`` header
+    # whose value matches ``$WITHCACHE_ADMIN_PASSWORD``, or a session
+    # cookie for browser-based operators.
+
+    def _bearer_or_session_authed(request: Request) -> None:
+        """Auth dependency for catalog writes.
+
+        - No password configured -> open (single-tenant LAN deploy).
+        - Session cookie present -> OK (browser path).
+        - Matching Bearer token -> OK (service-to-service path;
+          bty reads ``$WITHCACHE_ADMIN_PASSWORD`` and posts it).
+        - Otherwise -> 401.
+        """
+        auth = request.app.state.auth
+        if not auth.enabled:
+            return
+        # Session cookie check -- Starlette's SessionMiddleware
+        # stores the flag under ``withcache_authed`` in
+        # ``request.session``. Mirrors the flag ``_app.py`` sets.
+        if request.session.get("withcache_authed"):
+            return
+        header = request.headers.get("Authorization") or ""
+        if header.startswith("Bearer ") and auth.check_bearer(header[len("Bearer ") :]):
+            return
+        raise HTTPException(status_code=401, detail="auth required")
+
+    @app.get("/catalog")
+    def list_catalog(request: Request) -> JSONResponse:
+        """Return the current catalog as JSON.
+
+        Open route: bty polls this from a sibling container without a
+        session (mirrors the ``/blob`` byte-serving surface's trust
+        model -- LAN-only, no auth on reads). Returns a snapshot of
+        the in-process ``CatalogState.entries`` plus provenance
+        metadata (URL, env pin, fetch timestamps) so clients can
+        detect staleness. No pagination -- the catalog is small
+        (dozens of entries at most).
+        """
+        cs: CatalogState = request.app.state.catalog
+        return JSONResponse(
+            {
+                "url": cs.url,
+                "env_url": cs.env_url,
+                "fetched_at": cs.fetched_at,
+                "last_error": cs.last_error,
+                "entries": [dict(e) for e in cs.entries],
+            }
+        )
+
+    @app.post("/catalog/entries", status_code=201)
+    def add_catalog_entry(
+        body: dict[str, Any],
+        request: Request,
+        _auth: None = Depends(_bearer_or_session_authed),
+    ) -> JSONResponse:
+        """Insert one catalog entry.
+
+        Body: ``{"name": "...", "src": "...", "format?", "arch?",
+        "sha256?", "size_bytes?", "resolved_src?", "description?"}``.
+        Only ``src`` is required; every other field is optional and
+        gets persisted through the ``catalog.toml`` round-trip.
+
+        409 when an entry with the same ``name`` already exists;
+        rejecting on duplicate name (not src) because the display
+        surface keys on name and two rows with the same name would
+        collide on the /ui/catalog table.
+
+        Rejects any unknown top-level key -- the emitter allowlists
+        keys anyway, but failing loud at the API boundary saves the
+        operator from wondering why their ``notes`` field vanished.
+        """
+        allowed = {
+            "name",
+            "src",
+            "resolved_src",
+            "format",
+            "arch",
+            "sha256",
+            "size_bytes",
+            "description",
+        }
+        if not isinstance(body, dict):
+            raise HTTPException(status_code=400, detail="body must be a JSON object")
+        unknown = set(body.keys()) - allowed
+        if unknown:
+            raise HTTPException(
+                status_code=400,
+                detail=f"unknown key(s) {sorted(unknown)}; allowed: {sorted(allowed)}",
+            )
+        name = body.get("name")
+        src = body.get("src")
+        if not isinstance(name, str) or not name.strip():
+            raise HTTPException(status_code=400, detail="name: non-empty string required")
+        if not isinstance(src, str) or not src.strip():
+            raise HTTPException(status_code=400, detail="src: non-empty string required")
+
+        cs: CatalogState = request.app.state.catalog
+        if any(e.get("name") == name for e in cs.entries):
+            raise HTTPException(
+                status_code=409,
+                detail=f"catalog entry with name={name!r} already exists",
+            )
+        entry = {k: v for k, v in body.items() if v is not None and v != ""}
+        cs.entries.append(entry)
+        _persist_catalog(cs)
+        return JSONResponse(entry, status_code=201)
+
+    @app.delete("/catalog/entries", status_code=204)
+    def delete_catalog_entry(
+        request: Request,
+        name: str | None = None,
+        _auth: None = Depends(_bearer_or_session_authed),
+    ) -> Response:
+        """Delete via ``?name=<name>`` query param (URL-safe;
+        avoids path-encoding TOML entry names).
+
+        204 on success. 404 when no entry with that name exists.
+        """
+        if not name or not name.strip():
+            raise HTTPException(status_code=400, detail="name: query param required")
+        cs: CatalogState = request.app.state.catalog
+        original = list(cs.entries)
+        cs.entries = [e for e in cs.entries if e.get("name") != name]
+        if len(cs.entries) == len(original):
+            raise HTTPException(
+                status_code=404,
+                detail=f"no catalog entry with name={name!r}",
+            )
+        _persist_catalog(cs)
+        return Response(status_code=204)
+
+
+def _persist_catalog(cs: CatalogState) -> None:
+    """Write the current in-memory catalog back to
+    ``<data_dir>/catalog.toml`` (when ``persist_path`` is set).
+
+    Silent no-op when ``persist_path`` is ``None`` -- tests inject
+    a stub CatalogState without disk backing and still exercise
+    the in-memory mutation."""
+    if cs.persist_path is None:
+        return
+    from pathlib import Path
+
+    Path(cs.persist_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(cs.persist_path).write_bytes(_serialise_catalog(cs.entries))