withcache 0.6.2__tar.gz → 0.6.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {withcache-0.6.2 → withcache-0.6.3}/PKG-INFO +15 -1
- {withcache-0.6.2 → withcache-0.6.3}/README.md +14 -0
- {withcache-0.6.2 → withcache-0.6.3}/shim/build.zig.zon +1 -1
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/__init__.py +1 -1
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/server.py +4 -5
- {withcache-0.6.2 → withcache-0.6.3}/tests/test_differential.py +1 -1
- {withcache-0.6.2 → withcache-0.6.3}/tests/test_oras.py +16 -10
- {withcache-0.6.2 → withcache-0.6.3}/tests/test_withcache.py +10 -12
- {withcache-0.6.2 → withcache-0.6.3}/.gitignore +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/LICENSE +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/deploy/Containerfile +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/deploy/compose.yml +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/hatch_build.py +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/pyproject.toml +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/shim/build.zig +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/shim/shim.zig +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/_shim.py +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/client.py +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/curlwithcache.py +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/oras.py +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/static/htmx.min.js +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/static/pico.min.css +0 -0
- {withcache-0.6.2 → withcache-0.6.3}/src/withcache/wgetwithcache.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: withcache
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.3
|
|
4
4
|
Summary: Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)
|
|
5
5
|
Project-URL: Homepage, https://github.com/safl/withcache
|
|
6
6
|
Author-email: "Simon A. F. Lund" <safl@safl.dk>
|
|
@@ -285,6 +285,20 @@ the probe records the miss and, in auto-fetch mode, enqueues the fill, so the
|
|
|
285
285
|
next call flips to the cache. The encoding is shared with the shims and server,
|
|
286
286
|
so consumers stay in lockstep with the cache-host.
|
|
287
287
|
|
|
288
|
+
### Pull an `oras://` artifact (oras + client together)
|
|
289
|
+
|
|
290
|
+
For a registry blob, pair `withcache.oras` (resolve the reference to a blob URL
|
|
291
|
+
+ bearer) with `client.serve_url` (prefer the cache when warm). A runnable
|
|
292
|
+
example is in [`examples/pull_oras_blob.py`](examples/pull_oras_blob.py):
|
|
293
|
+
|
|
294
|
+
```python
|
|
295
|
+
from withcache import client, oras
|
|
296
|
+
|
|
297
|
+
resolved = oras.resolve_ref("oras://ghcr.io/<owner>/<repo>@sha256:<digest>")
|
|
298
|
+
url, headers = resolved.blob_url, dict(resolved.headers)
|
|
299
|
+
url = client.serve_url(server, url, headers=headers) or url # cache when warm
|
|
300
|
+
```
|
|
301
|
+
|
|
288
302
|
## Tests
|
|
289
303
|
|
|
290
304
|
```sh
|
|
@@ -267,6 +267,20 @@ the probe records the miss and, in auto-fetch mode, enqueues the fill, so the
|
|
|
267
267
|
next call flips to the cache. The encoding is shared with the shims and server,
|
|
268
268
|
so consumers stay in lockstep with the cache-host.
|
|
269
269
|
|
|
270
|
+
### Pull an `oras://` artifact (oras + client together)
|
|
271
|
+
|
|
272
|
+
For a registry blob, pair `withcache.oras` (resolve the reference to a blob URL
|
|
273
|
+
+ bearer) with `client.serve_url` (prefer the cache when warm). A runnable
|
|
274
|
+
example is in [`examples/pull_oras_blob.py`](examples/pull_oras_blob.py):
|
|
275
|
+
|
|
276
|
+
```python
|
|
277
|
+
from withcache import client, oras
|
|
278
|
+
|
|
279
|
+
resolved = oras.resolve_ref("oras://ghcr.io/<owner>/<repo>@sha256:<digest>")
|
|
280
|
+
url, headers = resolved.blob_url, dict(resolved.headers)
|
|
281
|
+
url = client.serve_url(server, url, headers=headers) or url # cache when warm
|
|
282
|
+
```
|
|
283
|
+
|
|
270
284
|
## Tests
|
|
271
285
|
|
|
272
286
|
```sh
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
.name = .withcache_shim,
|
|
3
3
|
// Zig requires a literal here; keep it in lockstep with the project's
|
|
4
4
|
// single source (src/withcache/__init__.py) via `make bump` / `make version-check`.
|
|
5
|
-
.version = "0.6.
|
|
5
|
+
.version = "0.6.3",
|
|
6
6
|
.fingerprint = 0xd7d96c5ed212ccaa,
|
|
7
7
|
.minimum_zig_version = "0.16.0",
|
|
8
8
|
.paths = .{
|
|
@@ -17,6 +17,6 @@ All modules are stdlib-only and self-contained.
|
|
|
17
17
|
from . import oras
|
|
18
18
|
from .client import blob_url, cache_base, is_cached, serve_url
|
|
19
19
|
|
|
20
|
-
__version__ = "0.6.
|
|
20
|
+
__version__ = "0.6.3"
|
|
21
21
|
|
|
22
22
|
__all__ = ["__version__", "blob_url", "cache_base", "is_cached", "oras", "serve_url"]
|
|
@@ -22,6 +22,7 @@ admin password is set, the operator UI is left open (with a startup warning).
|
|
|
22
22
|
|
|
23
23
|
import argparse
|
|
24
24
|
import base64
|
|
25
|
+
import contextlib
|
|
25
26
|
import hashlib
|
|
26
27
|
import hmac
|
|
27
28
|
import html
|
|
@@ -40,6 +41,7 @@ import urllib.parse
|
|
|
40
41
|
import urllib.request
|
|
41
42
|
from dataclasses import dataclass, field
|
|
42
43
|
from datetime import datetime, timezone
|
|
44
|
+
from typing import ClassVar
|
|
43
45
|
|
|
44
46
|
from . import __version__, oras
|
|
45
47
|
|
|
@@ -304,10 +306,8 @@ class Store:
|
|
|
304
306
|
"""Drop a cached artifact (row + bytes). The manual half of eviction."""
|
|
305
307
|
with _DB_WRITE_LOCK, self.conn() as c:
|
|
306
308
|
c.execute("DELETE FROM blobs WHERE key=?", (key,))
|
|
307
|
-
|
|
309
|
+
with contextlib.suppress(FileNotFoundError):
|
|
308
310
|
os.remove(self.blob_path(key))
|
|
309
|
-
except FileNotFoundError:
|
|
310
|
-
pass
|
|
311
311
|
|
|
312
312
|
def store_from_origin(
|
|
313
313
|
self,
|
|
@@ -471,7 +471,6 @@ class Store:
|
|
|
471
471
|
# --------------------------------------------------------------------------
|
|
472
472
|
# Background download manager (thread pool; modelled on bty's job managers)
|
|
473
473
|
# --------------------------------------------------------------------------
|
|
474
|
-
JOB_STATES = ("queued", "running", "completed", "cancelled", "failed")
|
|
475
474
|
PENDING_STATES = frozenset(("queued", "running"))
|
|
476
475
|
|
|
477
476
|
|
|
@@ -1012,7 +1011,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
|
|
1012
1011
|
self.end_headers()
|
|
1013
1012
|
|
|
1014
1013
|
# -- HTML --------------------------------------------------------------
|
|
1015
|
-
STATUS_COLORS = {
|
|
1014
|
+
STATUS_COLORS: ClassVar[dict[str, str]] = {
|
|
1016
1015
|
"queued": "#888",
|
|
1017
1016
|
"running": "var(--pico-primary, #0172ad)",
|
|
1018
1017
|
"completed": "#2e7d32",
|
|
@@ -15,7 +15,7 @@ import unittest
|
|
|
15
15
|
HERE = os.path.dirname(__file__)
|
|
16
16
|
sys.path.insert(0, os.path.join(HERE, "..", "src"))
|
|
17
17
|
|
|
18
|
-
from withcache import _shim
|
|
18
|
+
from withcache import _shim
|
|
19
19
|
|
|
20
20
|
SHIM = os.environ.get(
|
|
21
21
|
"WITHCACHE_SHIM", os.path.join(HERE, "..", "shim", "zig-out", "bin", "withcache-shim")
|
|
@@ -21,7 +21,7 @@ from unittest.mock import patch
|
|
|
21
21
|
|
|
22
22
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
|
23
23
|
|
|
24
|
-
from withcache import oras, server
|
|
24
|
+
from withcache import oras, server
|
|
25
25
|
|
|
26
26
|
# A trimmed-down version of a real nosi manifest -- two layers (the
|
|
27
27
|
# .img.gz and a .sha256 sidecar), one annotation each, OCI media types.
|
|
@@ -304,9 +304,11 @@ class TestResolveRef(unittest.TestCase):
|
|
|
304
304
|
def _failing(req, timeout=None):
|
|
305
305
|
raise OSError("network unreachable")
|
|
306
306
|
|
|
307
|
-
with
|
|
308
|
-
|
|
309
|
-
|
|
307
|
+
with (
|
|
308
|
+
patch("urllib.request.urlopen", _failing),
|
|
309
|
+
self.assertRaisesRegex(oras.OrasError, "token fetch failed"),
|
|
310
|
+
):
|
|
311
|
+
oras.resolve_ref("oras://ghcr.io/safl/nosi/debian-sysdev:latest")
|
|
310
312
|
|
|
311
313
|
|
|
312
314
|
class TestIsOrasUrl(unittest.TestCase):
|
|
@@ -341,9 +343,11 @@ class TestUrlopenRetry(unittest.TestCase):
|
|
|
341
343
|
calls["n"] += 1
|
|
342
344
|
raise _http_error(404)
|
|
343
345
|
|
|
344
|
-
with
|
|
345
|
-
|
|
346
|
-
|
|
346
|
+
with (
|
|
347
|
+
patch("urllib.request.urlopen", _notfound),
|
|
348
|
+
self.assertRaises(urllib.error.HTTPError),
|
|
349
|
+
):
|
|
350
|
+
oras._urlopen_retry("https://x/y", timeout=5)
|
|
347
351
|
self.assertEqual(calls["n"], 1)
|
|
348
352
|
|
|
349
353
|
def test_exhausts_then_reraises(self):
|
|
@@ -354,9 +358,11 @@ class TestUrlopenRetry(unittest.TestCase):
|
|
|
354
358
|
calls["n"] += 1
|
|
355
359
|
raise _http_error(503)
|
|
356
360
|
|
|
357
|
-
with
|
|
358
|
-
|
|
359
|
-
|
|
361
|
+
with (
|
|
362
|
+
patch("urllib.request.urlopen", _down),
|
|
363
|
+
self.assertRaises(urllib.error.HTTPError),
|
|
364
|
+
):
|
|
365
|
+
oras._urlopen_retry("https://x/y", timeout=5)
|
|
360
366
|
self.assertEqual(calls["n"], oras._RETRY_ATTEMPTS)
|
|
361
367
|
|
|
362
368
|
|
|
@@ -4,7 +4,9 @@ No third-party test deps; src/ is put on the path so the package imports
|
|
|
4
4
|
without an install.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
+
import contextlib
|
|
7
8
|
import http.server
|
|
9
|
+
import itertools
|
|
8
10
|
import os
|
|
9
11
|
import shutil
|
|
10
12
|
import socket
|
|
@@ -18,11 +20,11 @@ import unittest
|
|
|
18
20
|
|
|
19
21
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
|
20
22
|
|
|
21
|
-
import base64
|
|
22
|
-
import urllib.error
|
|
23
|
-
import urllib.request
|
|
23
|
+
import base64
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.request
|
|
24
26
|
|
|
25
|
-
from withcache import _shim, client, curlwithcache, server, wgetwithcache
|
|
27
|
+
from withcache import _shim, client, curlwithcache, server, wgetwithcache
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
# --------------------------------------------------------------------------
|
|
@@ -187,10 +189,8 @@ class _TruncatingOrigin(http.server.BaseHTTPRequestHandler):
|
|
|
187
189
|
# close the underlying socket so urllib observes EOF before
|
|
188
190
|
# Content-Length bytes arrive
|
|
189
191
|
self.wfile.flush()
|
|
190
|
-
|
|
192
|
+
with contextlib.suppress(OSError):
|
|
191
193
|
self.connection.shutdown(socket.SHUT_RDWR)
|
|
192
|
-
except OSError:
|
|
193
|
-
pass
|
|
194
194
|
|
|
195
195
|
def log_message(self, format, *args):
|
|
196
196
|
pass
|
|
@@ -307,10 +307,8 @@ class _ResumableTruncatingOrigin(http.server.BaseHTTPRequestHandler):
|
|
|
307
307
|
half = full // 2
|
|
308
308
|
self.wfile.write(self.PAYLOAD[:half])
|
|
309
309
|
self.wfile.flush()
|
|
310
|
-
|
|
310
|
+
with contextlib.suppress(OSError):
|
|
311
311
|
self.connection.shutdown(socket.SHUT_RDWR)
|
|
312
|
-
except OSError:
|
|
313
|
-
pass
|
|
314
312
|
else:
|
|
315
313
|
# any non-ranged retry serves the whole thing (covers the
|
|
316
314
|
# 200-on-Range fallback path: origin ignored Range, we
|
|
@@ -362,7 +360,7 @@ class TestRangeResumeOnTruncation(unittest.TestCase):
|
|
|
362
360
|
# final report should be the full payload
|
|
363
361
|
self.assertEqual(observed[-1][0], len(_ResumableTruncatingOrigin.PAYLOAD))
|
|
364
362
|
# at no point did the byte counter regress
|
|
365
|
-
for prev, curr in
|
|
363
|
+
for prev, curr in itertools.pairwise(observed):
|
|
366
364
|
self.assertGreaterEqual(curr[0], prev[0])
|
|
367
365
|
# the resume actually crossed the cut point: at least one
|
|
368
366
|
# progress call lands above the half-mark (otherwise we
|
|
@@ -732,7 +730,7 @@ class TestAutoFetchOnMiss(unittest.TestCase):
|
|
|
732
730
|
urllib.request.urlopen(_shim.blob_url(base, self.origin_url))
|
|
733
731
|
|
|
734
732
|
def test_miss_schedules_pull_by_default(self):
|
|
735
|
-
httpd,
|
|
733
|
+
httpd, _store = _start_withcache(auto_fetch=True)
|
|
736
734
|
try:
|
|
737
735
|
self._miss(httpd)
|
|
738
736
|
# the miss enqueued a background pull, no operator needed
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|