withcache 0.6.5__tar.gz → 0.7.0__tar.gz

{withcache-0.6.5 → withcache-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: withcache
-Version: 0.6.5
+Version: 0.7.0
 Summary: Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)
 Project-URL: Homepage, https://github.com/safl/withcache
 Author-email: "Simon A. F. Lund" <safl@safl.dk>
@@ -13,7 +13,7 @@ Classifier: Intended Audience :: System Administrators
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: System :: Archiving :: Mirroring
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
 # withcache

{withcache-0.6.5 → withcache-0.7.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "withcache"
 dynamic = ["version"]  # single source of truth: src/withcache/__init__.py:__version__
 description = "Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = { text = "BSD-3-Clause" }
 authors = [{ name = "Simon A. F. Lund", email = "safl@safl.dk" }]
 keywords = ["cache", "artifacts", "cuda", "rocm", "doca", "firmware", "lab"]
@@ -45,7 +45,7 @@ include = ["src/withcache", "shim", "README.md", "LICENSE", "deploy", "tests"]
 
 [tool.ruff]
 line-length = 100
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "UP", "B"]

{withcache-0.6.5 → withcache-0.7.0}/shim/build.zig.zon

@@ -2,7 +2,7 @@
     .name = .withcache_shim,
     // Zig requires a literal here; keep it in lockstep with the project's
     // single source (src/withcache/__init__.py) via `make bump` / `make version-check`.
-    .version = "0.6.5",
+    .version = "0.7.0",
     .fingerprint = 0xd7d96c5ed212ccaa,
     .minimum_zig_version = "0.16.0",
     .paths = .{

{withcache-0.6.5 → withcache-0.7.0}/src/withcache/__init__.py

@@ -17,6 +17,6 @@ All modules are stdlib-only and self-contained.
 from . import oras
 from .client import blob_url, cache_base, is_cached, serve_url
 
-__version__ = "0.6.5"
+__version__ = "0.7.0"
 
 __all__ = ["__version__", "blob_url", "cache_base", "is_cached", "oras", "serve_url"]

{withcache-0.6.5 → withcache-0.7.0}/src/withcache/server.py

@@ -37,11 +37,13 @@ import socketserver
 import sqlite3
 import threading
 import time
+import tomllib
+import urllib.error
 import urllib.parse
 import urllib.request
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
-from typing import ClassVar
+from datetime import UTC, datetime
+from typing import Any, ClassVar
 
 from . import __version__, oras
 
@@ -70,7 +72,7 @@ _DB_WRITE_LOCK = threading.Lock()
 
 
 def now_iso() -> str:
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
 def _age_human(started_at: float, *, now: float | None = None) -> str:
@@ -147,6 +149,84 @@ def resolve_secret(data_dir: str) -> bytes:
     return secret
 
 
+DEFAULT_CATALOG_URL = "https://github.com/safl/nosi/releases/latest/download/catalog.toml"
+
+
+@dataclass
+class CatalogState:
+    """Live state of the fetched image catalog.
+
+    withcache does not run a background poller. The catalog is
+    fetched at process start (best-effort, persisted result from
+    the last successful fetch survives a restart), refetched
+    lazily when the dashboard renders and the in-memory copy is
+    older than :data:`CATALOG_CACHE_TTL`, and force-refetched by
+    the operator via the Refresh button (POST /admin/catalog_refresh).
+
+    The raw TOML bytes are persisted to ``<data_dir>/catalog.toml``
+    so a restart doesn't wipe the last known good catalog and the
+    same file can be re-served verbatim to consumers on a future
+    ``GET /catalog.toml`` route.
+    """
+
+    url: str
+    persist_path: str
+    entries: list[dict[str, Any]] = field(default_factory=list)
+    fetched_at: str = ""
+    last_error: str = ""
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def load_persisted(self) -> None:
+        """Best-effort: seed ``entries`` + ``fetched_at`` from the on-disk
+        ``catalog.toml`` if it exists. Never raises; a corrupt or missing
+        file leaves the state empty so the operator sees the "not
+        fetched yet" hint in the dashboard."""
+        if not os.path.isfile(self.persist_path):
+            return
+        try:
+            with open(self.persist_path, "rb") as f:
+                raw = f.read()
+            parsed = tomllib.loads(raw.decode("utf-8"))
+            mtime = os.path.getmtime(self.persist_path)
+            self.entries = list(parsed.get("images") or [])
+            self.fetched_at = datetime.fromtimestamp(mtime, UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+        except (OSError, ValueError, UnicodeDecodeError, tomllib.TOMLDecodeError) as e:
+            self.last_error = f"failed to load {self.persist_path}: {e}"
+
+    def fetch_now(self, *, timeout: float = 15.0) -> None:
+        """Fetch the catalog URL, parse, persist raw bytes, populate
+        ``entries``. On failure the previously-cached entries remain;
+        the error is recorded so the tab can surface it. Single-writer
+        via ``self._lock`` so a burst of clicks doesn't double-fetch."""
+        with self._lock:
+            try:
+                req = urllib.request.Request(self.url, headers={"User-Agent": USER_AGENT})
+                with urllib.request.urlopen(req, timeout=timeout) as resp:
+                    raw = resp.read()
+                # Validate before persisting -- a 500 from upstream that
+                # returns HTML must not clobber a previously-good
+                # catalog.toml on disk.
+                parsed = tomllib.loads(raw.decode("utf-8"))
+                entries = list(parsed.get("images") or [])
+                # Atomic write: tempfile + rename so a crash mid-write
+                # never leaves a half-written catalog.toml.
+                tmp = self.persist_path + ".tmp"
+                with open(tmp, "wb") as f:
+                    f.write(raw)
+                os.replace(tmp, self.persist_path)
+                self.entries = entries
+                self.fetched_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+                self.last_error = ""
+            except (
+                urllib.error.URLError,
+                OSError,
+                ValueError,
+                UnicodeDecodeError,
+                tomllib.TOMLDecodeError,
+            ) as e:
+                self.last_error = str(e)
+
+
 class Auth:
     COOKIE = "withcache-token"
     MAX_AGE = 7 * 24 * 3600  # cookie lifetime, seconds
@@ -745,6 +825,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
     def streams(self) -> StreamRegistry:
         return self.server.streams  # type: ignore[attr-defined]
 
+    @property
+    def catalog(self) -> CatalogState:
+        return self.server.catalog  # type: ignore[attr-defined]
+
     def log_message(self, format, *args):  # quieter, single-line
         print(f"{self.address_string()} - {format % args}", flush=True)
 
@@ -793,6 +877,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
         "/admin/delete",
         "/admin/cancel",
         "/admin/clear",
+        "/admin/catalog_refresh",
     )
 
     def do_POST(self):
@@ -820,6 +905,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
                     self.mgr.cancel(int(jid))
             elif parsed.path == "/admin/clear":
                 self.mgr.clear_finished()
+            elif parsed.path == "/admin/catalog_refresh":
+                self.catalog.fetch_now()
             self.respond_admin()
         else:
             self.send_text(404, "not found\n")
@@ -1354,6 +1441,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
         nstreams = len(streams)
         njobs = len(jobs)
 
+        catalog_rows = self._catalog_rows()
+
         return f"""
   <p class="text-muted small mb-2">{nblobs} cached ({used}){full}
     &middot; {nmisses} pending miss(es)</p>
@@ -1367,6 +1456,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
         href="#tab-downloads">Downloads ({njobs})</a></li>
       <li class="nav-item"><a class="nav-link {_active("tab-misses").lstrip()}"
         href="#tab-misses">Misses ({nmisses})</a></li>
+      <li class="nav-item"><a class="nav-link {_active("tab-catalog").lstrip()}"
+        href="#tab-catalog">Catalog ({len(self.catalog.entries)})</a></li>
     </ul>
   </nav>
 
@@ -1412,8 +1503,80 @@ class Handler(http.server.BaseHTTPRequestHandler):
       </tr></thead>
       <tbody>{miss_rows}</tbody>
     </table></div>
+  </section>
+
+  <section id="tab-catalog" class="tab{_active("tab-catalog")}">
+    <div class="d-flex align-items-center justify-content-between mb-2 flex-wrap gap-2">
+      <div>
+        <div><small class="text-muted">source</small>
+          &nbsp;<code>{html.escape(self.catalog.url)}</code></div>
+        <div><small class="text-muted">fetched</small>
+          &nbsp;<code>{html.escape(self.catalog.fetched_at) or "never"}</code>
+          {
+            (
+                f'<span class="badge bg-danger-subtle text-danger ms-1">'
+                f"{html.escape(self.catalog.last_error)}</span>"
+            )
+            if self.catalog.last_error
+            else ""
+        }</div>
+      </div>
+      <form hx-post="/admin/catalog_refresh" hx-target="#dash" hx-swap="innerHTML"
+            hx-indicator="#spin" class="m-0">
+        <button class="btn btn-sm btn-primary" type="submit">Refresh</button>
+      </form>
+    </div>
+    <div class="table-responsive"><table class="table table-sm table-striped table-hover mb-0">
+      <thead class="table-light"><tr>
+        <th>Name</th><th>Format</th><th>Arch</th><th>Size</th><th>Source</th>
+      </tr></thead>
+      <tbody>{catalog_rows}</tbody>
+    </table></div>
   </section>"""
 
+    def _catalog_rows(self) -> str:
+        """One <tr> per catalog image. Cells: name, format, arch,
+        human-readable size, and a Source cell whose URL is the
+        withcache /b/ token (so a click through pulls via cache).
+        Empty catalog rows to a "not fetched yet" placeholder + a
+        hint about the Refresh button."""
+        entries = self.catalog.entries
+        if not entries:
+            hint = self.catalog.last_error or "click Refresh above to fetch the catalog manifest"
+            return (
+                '<tr><td colspan="5" class="text-center text-muted">'
+                f"<em>{html.escape(hint)}</em></td></tr>"
+            )
+        rows: list[str] = []
+        for entry in entries:
+            name = str(entry.get("name") or "")
+            src = str(entry.get("src") or "")
+            fmt = str(entry.get("format") or "")
+            arch = str(entry.get("arch") or "")
+            size_bytes = entry.get("size_bytes")
+            size = human_size(int(size_bytes)) if isinstance(size_bytes, int) else ""
+            # Route the Source column via /b/<token>/<name> so a
+            # click serves through withcache -- first hit fills the
+            # cache, subsequent hits serve from disk. Mirrors the
+            # blob-row rendering above.
+            if src.startswith(("http://", "https://", "oras://")):
+                token = base64.urlsafe_b64encode(src.encode("utf-8")).decode("ascii").rstrip("=")
+                filename = urllib.parse.urlsplit(src).path.rsplit("/", 1)[-1] or "download"
+                link = f"/b/{token}/{urllib.parse.quote(filename)}"
+                src_cell = f'<a href="{link}" title="{html.escape(src)}">{html.escape(src)}</a>'
+            else:
+                src_cell = html.escape(src)
+            rows.append(
+                f"<tr>"
+                f"<td>{html.escape(name)}</td>"
+                f'<td class="mono"><small>{html.escape(fmt)}</small></td>'
+                f'<td class="mono"><small>{html.escape(arch)}</small></td>'
+                f"<td>{html.escape(size)}</td>"
+                f'<td class="url">{src_cell}</td>'
+                "</tr>"
+            )
+        return "".join(rows)
+
     def _stream_progress_cell(self, s: Stream) -> str:
         """One progress cell for an active stream: a <progress> bar when the
         total is known (always for a cached blob, since the size came off
@@ -1506,6 +1669,25 @@ def main():
     httpd.mgr = mgr  # type: ignore[attr-defined]
     httpd.auto_fetch = not args.curate  # type: ignore[attr-defined]
     httpd.streams = StreamRegistry()  # type: ignore[attr-defined]
+    # Catalog state: env override, else the shipping default (nosi's
+    # rolling catalog manifest). Seeded from the last persisted
+    # catalog.toml on disk so a restart doesn't wipe the cache.
+    catalog_url = (os.environ.get("WITHCACHE_CATALOG_URL") or DEFAULT_CATALOG_URL).strip()
+    catalog = CatalogState(
+        url=catalog_url,
+        persist_path=os.path.join(store.data_dir, "catalog.toml"),
+    )
+    catalog.load_persisted()
+    httpd.catalog = catalog  # type: ignore[attr-defined]
+    # If no catalog is persisted yet, kick a single startup fetch
+    # in a daemon thread so the operator has something to look at
+    # without needing to click Refresh on their first visit.
+    # Failures record ``last_error`` in memory; the dashboard row
+    # shows it. Never blocks the serve loop.
+    if not catalog.entries:
+        threading.Thread(
+            target=catalog.fetch_now, name="withcache-catalog-init", daemon=True
+        ).start()
     print(
         f"withcache cache-host on http://{args.host}:{args.port}  "
         f"(data={store.data_dir}, keep_query={args.keep_query}, workers={args.workers}, "

{withcache-0.6.5 → withcache-0.7.0}/tests/test_withcache.py

@@ -643,6 +643,14 @@ def _start_withcache(auto_fetch=False):
     httpd.mgr = server.DownloadManager(store, workers=1)
     httpd.auto_fetch = auto_fetch
     httpd.streams = server.StreamRegistry()
+    # Empty catalog with a bogus URL so the tab renders "click Refresh
+    # above to fetch the catalog manifest" instead of firing a real
+    # HTTP call. Tests that specifically exercise the catalog code
+    # can override this attribute after the helper returns.
+    httpd.catalog = server.CatalogState(
+        url="http://localhost/catalog.toml",
+        persist_path=os.path.join(store.data_dir, "catalog.toml"),
+    )
     threading.Thread(target=httpd.serve_forever, daemon=True).start()
     return httpd, store