withcache 0.3.0__tar.gz → 0.4.1__tar.gz

{withcache-0.3.0 → withcache-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: withcache
-Version: 0.3.0
+Version: 0.4.1
 Summary: Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)
 Project-URL: Homepage, https://github.com/safl/withcache
 Author-email: "Simon A. F. Lund" <safl@safl.dk>
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 
 # withcache
 
-[![ci](https://github.com/safl/withcache/actions/workflows/ci.yml/badge.svg)](https://github.com/safl/withcache/actions/workflows/ci.yml)
+[![ci](https://github.com/safl/withcache/actions/workflows/ci-cd.yml/badge.svg)](https://github.com/safl/withcache/actions/workflows/ci-cd.yml)
 [![PyPI](https://img.shields.io/pypi/v/withcache.svg)](https://pypi.org/project/withcache/)
 [![license](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)](LICENSE)
 [![built with Zig](https://img.shields.io/badge/built%20with-Zig%200.16.0-f7a41d.svg)](https://ziglang.org)

{withcache-0.3.0 → withcache-0.4.1}/README.md

@@ -1,6 +1,6 @@
 # withcache
 
-[![ci](https://github.com/safl/withcache/actions/workflows/ci.yml/badge.svg)](https://github.com/safl/withcache/actions/workflows/ci.yml)
+[![ci](https://github.com/safl/withcache/actions/workflows/ci-cd.yml/badge.svg)](https://github.com/safl/withcache/actions/workflows/ci-cd.yml)
 [![PyPI](https://img.shields.io/pypi/v/withcache.svg)](https://pypi.org/project/withcache/)
 [![license](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)](LICENSE)
 [![built with Zig](https://img.shields.io/badge/built%20with-Zig%200.16.0-f7a41d.svg)](https://ziglang.org)

{withcache-0.3.0 → withcache-0.4.1}/shim/build.zig.zon

@@ -2,7 +2,7 @@
     .name = .withcache_shim,
     // Zig requires a literal here; keep it in lockstep with the project's
     // single source (src/withcache/__init__.py) via `make bump` / `make version-check`.
-    .version = "0.3.0",
+    .version = "0.4.1",
     .fingerprint = 0xd7d96c5ed212ccaa,
     .minimum_zig_version = "0.16.0",
     .paths = .{

{withcache-0.3.0 → withcache-0.4.1}/src/withcache/__init__.py

@@ -12,6 +12,6 @@ All modules are stdlib-only and self-contained.
 
 from .client import blob_url, cache_base, is_cached, serve_url
 
-__version__ = "0.3.0"
+__version__ = "0.4.1"
 
 __all__ = ["__version__", "blob_url", "cache_base", "is_cached", "serve_url"]

{withcache-0.3.0 → withcache-0.4.1}/src/withcache/client.py

@@ -37,13 +37,31 @@ def blob_url(server: str, origin: str) -> str:
     return _shim.blob_url(_shim.cache_base(server), origin)
 
 
-def is_cached(server: str, origin: str, timeout: float = PROBE_TIMEOUT) -> bool:
+def is_cached(
+    server: str,
+    origin: str,
+    timeout: float = PROBE_TIMEOUT,
+    headers: dict[str, str] | None = None,
+) -> bool:
     """True if the cache-host already holds ``origin`` (a ``HEAD`` on ``/b/``
     returns 200). A miss (404), an unreachable host, a timeout, or any error
     returns False, so a caller can safely fall back to the origin. The HEAD
     also *warms* an auto-fetch cache-host: the miss is recorded and the
-    background fill enqueued, so a later probe flips to cached."""
+    background fill enqueued, so a later probe flips to cached.
+
+    ``headers`` (optional) attaches request headers to the HEAD. The
+    cache-host forwards a client-supplied ``Authorization`` into its
+    background-fetch worker, so a consumer that has just minted an OCI
+    bearer (the typical use case: bty resolving an ``oras://`` catalog
+    entry to a ``ghcr.io`` blob URL at import time) can warm the cache
+    against that token-gated origin in one probe. Other entries in
+    ``headers`` round-trip the same way; only ``Authorization`` is
+    forwarded into the fetch on the server side.
+    """
     req = urllib.request.Request(blob_url(server, origin), method="HEAD")
+    if headers:
+        for k, v in headers.items():
+            req.add_header(k, v)
     try:
         with urllib.request.urlopen(req, timeout=timeout) as resp:
             return bool(resp.status == 200)
@@ -53,10 +71,19 @@ def is_cached(server: str, origin: str, timeout: float = PROBE_TIMEOUT) -> bool:
         return False  # unreachable / timeout -> caller serves the origin itself
 
 
-def serve_url(server: str, origin: str, timeout: float = PROBE_TIMEOUT) -> str | None:
+def serve_url(
+    server: str,
+    origin: str,
+    timeout: float = PROBE_TIMEOUT,
+    headers: dict[str, str] | None = None,
+) -> str | None:
     """The cache-host serve URL for ``origin`` if the cache holds it, else
     ``None`` -- the convenience form of "use the cache when warm":
 
         url = client.serve_url(cache, origin) or origin
+
+    ``headers`` is passed through to :func:`is_cached` for the HEAD probe;
+    the returned serve URL never carries auth (cached bytes are served
+    without revisiting the origin).
     """
-    return blob_url(server, origin) if is_cached(server, origin, timeout) else None
+    return blob_url(server, origin) if is_cached(server, origin, timeout, headers=headers) else None

{withcache-0.3.0 → withcache-0.4.1}/src/withcache/server.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
-"""withcache cache-host — an operator-curated artifact cache.
+"""withcache cache-host — a URL-keyed artifact cache.
 
 Stdlib only (http.server + sqlite3 + urllib). Serves cached blobs keyed by
-their origin URL. A cache miss is *not* fetched automatically: it is recorded
-in a miss table so an operator can review it and press "Download", at which
-point the cache-host pulls the artifact from origin and stores it. There is
-also an "add from URI" form to pre-seed an artifact before anyone misses it.
+their origin URL. By default a cache miss is auto-fetched: it is recorded in the
+miss table and pulled from origin in the background, so the next request hits
+(the client falls through to origin on the first miss). Run with `--curate` to
+require an operator to approve each pull via a small web UI instead; either way
+you can pre-seed an artifact with the "Add from URI" form.
 
 This is the only component that needs internet egress (and any vendor creds).
 Clients never write to it.
 
-Auth (modelled on bty's single-tenant approach, minus PAM): the read path
+Auth (single-tenant: env password + signed cookie): the read path
 (`/blob`, `/healthz`) is open so clients never log in; the operator surface
 (`/` and `/admin/*`) is gated behind a server-signed session cookie. Login at
 `POST /ui/login` checks the password in $WITHCACHE_ADMIN_PASSWORD and flips the
@@ -324,6 +325,17 @@ class Store:
                         size += len(chunk)
                         if progress:
                             progress(size, total)
+            # urllib's read loop exits on clean EOF AND on transport-
+            # aborted close; HTTPResponse only raises IncompleteRead
+            # in some configurations. When the origin declared
+            # Content-Length, treat that as the contract and refuse
+            # to promote a short blob. A silent partial-promotion
+            # would serve malformed bytes to every future consumer
+            # with no way for them to invalidate the entry.
+            if total is not None and size != total:
+                raise TruncatedDownload(
+                    f"upstream truncated for {url}: declared {total} bytes, got {size}"
+                )
             os.replace(tmp, self.blob_path(key))
         except BaseException:
             if os.path.exists(tmp):
@@ -368,6 +380,14 @@ class CacheFull(Exception):
     """Raised when --max-bytes is reached; the fill is refused, not evicted."""
 
 
+class TruncatedDownload(Exception):
+    """Raised when the upstream stream ended before the declared
+    Content-Length. The temp file is removed and no blob row is
+    written, so the same URL re-enqueues cleanly on the next request
+    instead of permanently serving a malformed file.
+    """
+
+
 @dataclass
 class Job:
     id: int
@@ -663,7 +683,21 @@ class Handler(http.server.BaseHTTPRequestHandler):
                 # falls through on a miss). In --curate mode an operator triggers
                 # the pull instead; when the cache is full we record the miss but
                 # schedule nothing (delete something first).
-                self.mgr.enqueue(url)
+                #
+                # Forward the client's ``Authorization`` header into the worker
+                # job so a token-gated origin (typical use case: a fresh OCI
+                # bearer on a ghcr.io blob URL minted by bty-web at catalog
+                # import time) can be fetched. Without this the worker runs
+                # anonymous and 401s; the URL stays uncached forever. Keep the
+                # allowlist narrow on purpose: ``Authorization`` is the only
+                # request header we proxy onto the worker. The ``/admin/fetch``
+                # operator endpoint still carries its own ``headers=`` payload
+                # for the curated path.
+                fwd_headers = None
+                auth = self.headers.get("Authorization")
+                if auth:
+                    fwd_headers = {"Authorization": auth}
+                self.mgr.enqueue(url, headers=fwd_headers)
             self.send_text(404, "cache miss (recorded)\n")
             return
         path = self.store.blob_path(row["key"])

{withcache-0.3.0 → withcache-0.4.1}/tests/test_withcache.py

@@ -7,10 +7,12 @@ without an install.
 import http.server
 import os
 import shutil
+import socket
 import socketserver
 import sys
 import tempfile
 import threading
+import time
 import unittest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
@@ -166,6 +168,78 @@ class TestStoreFromOrigin(unittest.TestCase):
             store.store_from_origin(f"http://127.0.0.1:{self.port}/b.bin")
 
 
+class _TruncatingOrigin(http.server.BaseHTTPRequestHandler):
+    """Declare a full Content-Length, then send half the payload and
+    close the socket. Mirrors the real-world failure mode where the
+    upstream drops the connection mid-stream (lab-box fedora-44-desktop
+    flash that surfaced this bug)."""
+
+    PAYLOAD = b"abcdefghij" * 100  # 1000 bytes; will write half then close
+
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(self.PAYLOAD)))
+        self.end_headers()
+        half = len(self.PAYLOAD) // 2
+        self.wfile.write(self.PAYLOAD[:half])
+        # close the underlying socket so urllib observes EOF before
+        # Content-Length bytes arrive
+        self.wfile.flush()
+        try:
+            self.connection.shutdown(socket.SHUT_RDWR)
+        except OSError:
+            pass
+
+    def log_message(self, format, *args):
+        pass
+
+
+class TestTruncatedDownloadRejected(unittest.TestCase):
+    """Regression for the lab-spotted bug where a transport-aborted
+    upstream stream silently became a permanent cached blob: future
+    HEADs returned 200 with the partial bytes, every consumer got a
+    malformed file, and the only escape was hand-deleting the blob.
+    Content-Length mismatches now fail loudly and leave no entry."""
+
+    def setUp(self):
+        self.httpd = socketserver.TCPServer(("127.0.0.1", 0), _TruncatingOrigin)
+        self.port = self.httpd.server_address[1]
+        self.t = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+        self.t.start()
+        self.store = server.Store(tempfile.mkdtemp(), keep_query=False)
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def test_truncated_upstream_raises_and_leaves_no_blob(self):
+        url = f"http://127.0.0.1:{self.port}/truncated.bin"
+        with self.assertRaises(server.TruncatedDownload) as cm:
+            self.store.store_from_origin(url)
+        # the message must name both totals so the operator can see
+        # how short the upstream came
+        msg = str(cm.exception)
+        self.assertIn("1000", msg)  # declared
+        self.assertIn("500", msg)  # got
+        # no row was written; no blob file lingers on disk
+        self.assertIsNone(self.store.get_blob(url))
+        blobs = list(self.store.blob_path("").rsplit("/", 1)[0:1])
+        if os.path.isdir(blobs[0]):
+            self.assertEqual(os.listdir(blobs[0]), [])
+
+    def test_repeat_request_after_truncation_can_retry_cleanly(self):
+        url = f"http://127.0.0.1:{self.port}/truncated.bin"
+        with self.assertRaises(server.TruncatedDownload):
+            self.store.store_from_origin(url)
+        # second attempt against the same URL would have hit the
+        # poisoned cache before the fix; now it must repeat the
+        # failure mode (no sticky blob blocking the retry) so a
+        # later origin recovery can re-fill the entry cleanly.
+        with self.assertRaises(server.TruncatedDownload):
+            self.store.store_from_origin(url)
+
+
 # --------------------------------------------------------------------------
 # _shim: URL detection, rewrite, real-tool resolution, env, path-encoding
 # --------------------------------------------------------------------------
@@ -482,6 +556,60 @@ class TestFetchWithHeaders(unittest.TestCase):
         self.assertEqual(row["size"], len(PAYLOAD))
 
 
+# --------------------------------------------------------------------------
+# HEAD with an Authorization header should propagate that header into the
+# auto-fetch worker so a 401-gated origin (e.g. a ghcr.io blob URL behind a
+# bty-minted OCI bearer) actually fills. Without this propagation the worker
+# pulls anonymous, the origin 401s, and the URL stays uncached forever.
+# --------------------------------------------------------------------------
+class TestHeadForwardsAuthorizationToAutoFetch(unittest.TestCase):
+    def setUp(self):
+        self.origin = socketserver.TCPServer(("127.0.0.1", 0), _AuthOrigin)
+        threading.Thread(target=self.origin.serve_forever, daemon=True).start()
+        self.origin_url = f"http://127.0.0.1:{self.origin.server_address[1]}/blob.bin"
+        self.httpd, self.store = _start_withcache(auto_fetch=True)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        for s in (self.origin, self.httpd):
+            s.shutdown()
+            s.server_close()
+
+    def _wait_for_fill(self, timeout_s=2.0):
+        deadline = time.monotonic() + timeout_s
+        while time.monotonic() < deadline:
+            if self.store.get_blob(self.origin_url) is not None:
+                return True
+            time.sleep(0.02)
+        return False
+
+    def test_head_with_authorization_triggers_authed_fetch(self):
+        bu = _shim.blob_url(self.base, self.origin_url)
+        req = urllib.request.Request(bu, method="HEAD")
+        req.add_header("Authorization", _AuthOrigin.TOKEN)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(req)
+        self.assertEqual(cm.exception.code, 404)  # miss; recorded + enqueued
+        # The worker should have fetched in the background using the header.
+        self.assertTrue(
+            self._wait_for_fill(),
+            "expected blob to be cached after auth-bearing HEAD",
+        )
+
+    def test_head_without_authorization_leaves_origin_401_and_cache_empty(self):
+        # Negative: no Authorization on the HEAD means the worker is enqueued
+        # anonymous, the origin 401s, nothing lands. Verifies the new code
+        # path is genuinely opt-in (HEAD without auth keeps the old behaviour).
+        bu = _shim.blob_url(self.base, self.origin_url)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(urllib.request.Request(bu, method="HEAD"))
+        self.assertEqual(cm.exception.code, 404)
+        self.assertFalse(
+            self._wait_for_fill(timeout_s=0.5),
+            "expected no blob without forwarded auth",
+        )
+
+
 # --------------------------------------------------------------------------
 # Pure helpers
 # --------------------------------------------------------------------------
@@ -539,5 +667,61 @@ class TestClientLibrary(unittest.TestCase):
         self.assertFalse(client.is_cached("http://127.0.0.1:9", self.origin_url, timeout=0.5))
 
 
+# --------------------------------------------------------------------------
+# Client + server end-to-end: a HEAD with ``headers={"Authorization": ...}``
+# warms the cache against a 401-gated origin. Mirrors the bty oras case
+# (resolved ghcr.io blob URL + freshly-minted OCI bearer).
+# --------------------------------------------------------------------------
+class TestClientLibraryAuthForwarding(unittest.TestCase):
+    def setUp(self):
+        self.origin = socketserver.TCPServer(("127.0.0.1", 0), _AuthOrigin)
+        threading.Thread(target=self.origin.serve_forever, daemon=True).start()
+        self.origin_url = f"http://127.0.0.1:{self.origin.server_address[1]}/blob.bin"
+        self.httpd, self.store = _start_withcache(auto_fetch=True)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        for s in (self.origin, self.httpd):
+            s.shutdown()
+            s.server_close()
+
+    def test_is_cached_with_authorization_warms_auth_gated_origin(self):
+        # Cold: no auth -> background fetch goes anonymous, 401s, cache empty.
+        self.assertFalse(client.is_cached(self.base, self.origin_url))
+        deadline = time.monotonic() + 0.5
+        while time.monotonic() < deadline:
+            if self.store.get_blob(self.origin_url) is not None:
+                break
+            time.sleep(0.02)
+        self.assertIsNone(self.store.get_blob(self.origin_url))
+
+        # Warm-with-token: HEAD carries Authorization; server forwards it
+        # into the fetch worker; the auth-gated origin returns the bytes.
+        self.assertFalse(
+            client.is_cached(
+                self.base,
+                self.origin_url,
+                headers={"Authorization": _AuthOrigin.TOKEN},
+            )
+        )
+        deadline = time.monotonic() + 2.0
+        while time.monotonic() < deadline:
+            if self.store.get_blob(self.origin_url) is not None:
+                break
+            time.sleep(0.02)
+        self.assertIsNotNone(
+            self.store.get_blob(self.origin_url),
+            "expected auth-bearing HEAD to fill the cache via forwarded Authorization",
+        )
+
+        # And once cached, the auth header is no longer needed: a plain HEAD
+        # hits 200, serve_url returns the blob URL without auth.
+        self.assertTrue(client.is_cached(self.base, self.origin_url))
+        self.assertEqual(
+            client.serve_url(self.base, self.origin_url),
+            client.blob_url(self.base, self.origin_url),
+        )
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)