withcache 0.5.2__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {withcache-0.5.2 → withcache-0.6.1}/PKG-INFO +1 -1
- {withcache-0.5.2 → withcache-0.6.1}/shim/build.zig.zon +1 -1
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/__init__.py +7 -2
- withcache-0.6.1/src/withcache/oras.py +543 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/server.py +90 -6
- withcache-0.6.1/tests/test_oras.py +480 -0
- {withcache-0.5.2 → withcache-0.6.1}/tests/test_withcache.py +62 -0
- {withcache-0.5.2 → withcache-0.6.1}/.gitignore +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/LICENSE +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/README.md +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/deploy/Containerfile +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/deploy/compose.yml +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/hatch_build.py +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/pyproject.toml +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/shim/build.zig +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/shim/shim.zig +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/_shim.py +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/client.py +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/curlwithcache.py +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/static/htmx.min.js +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/static/pico.min.css +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/src/withcache/wgetwithcache.py +0 -0
- {withcache-0.5.2 → withcache-0.6.1}/tests/test_differential.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: withcache
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Operator-curated, URL-keyed artifact cache for a small lab (CUDA/ROCm/DOCA/firmware)
|
|
5
5
|
Project-URL: Homepage, https://github.com/safl/withcache
|
|
6
6
|
Author-email: "Simon A. F. Lund" <safl@safl.dk>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
.name = .withcache_shim,
|
|
3
3
|
// Zig requires a literal here; keep it in lockstep with the project's
|
|
4
4
|
// single source (src/withcache/__init__.py) via `make bump` / `make version-check`.
|
|
5
|
-
.version = "0.
|
|
5
|
+
.version = "0.6.1",
|
|
6
6
|
.fingerprint = 0xd7d96c5ed212ccaa,
|
|
7
7
|
.minimum_zig_version = "0.16.0",
|
|
8
8
|
.paths = .{
|
|
@@ -6,12 +6,17 @@
|
|
|
6
6
|
- ``withcache.client``: a tiny, stdlib-only library for other tools to consume
|
|
7
7
|
a cache-host (build serve URLs, probe what's cached) without re-implementing
|
|
8
8
|
the ``/b/`` URL scheme.
|
|
9
|
+
- ``withcache.oras``: OCI registry adapter. Parses ``oras://...`` references
|
|
10
|
+
and resolves them to a plain HTTPS blob URL + bearer headers. The cache-host
|
|
11
|
+
uses it on a cold miss; library consumers (e.g. ``bty``) import it to
|
|
12
|
+
validate catalog entries and pre-resolve content digests.
|
|
9
13
|
|
|
10
14
|
All modules are stdlib-only and self-contained.
|
|
11
15
|
"""
|
|
12
16
|
|
|
17
|
+
from . import oras
|
|
13
18
|
from .client import blob_url, cache_base, is_cached, serve_url
|
|
14
19
|
|
|
15
|
-
__version__ = "0.
|
|
20
|
+
__version__ = "0.6.1"
|
|
16
21
|
|
|
17
|
-
__all__ = ["__version__", "blob_url", "cache_base", "is_cached", "serve_url"]
|
|
22
|
+
__all__ = ["__version__", "blob_url", "cache_base", "is_cached", "oras", "serve_url"]
|
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
"""ORAS / OCI registry adapter for fetching disk images and other artifacts.
|
|
2
|
+
|
|
3
|
+
Resolves ``oras://`` references against an OCI v2 registry into a plain
|
|
4
|
+
HTTPS blob URL plus the auth headers the registry expects. The cache-host
|
|
5
|
+
(``withcache.server``) calls in here on a cold miss when the decoded
|
|
6
|
+
origin URL is ``oras://...`` so the existing Range-resume fetch loop
|
|
7
|
+
can stream the bytes; library consumers (e.g. ``bty``) import the same
|
|
8
|
+
helpers to validate catalog entries and pre-resolve content digests.
|
|
9
|
+
|
|
10
|
+
URL shape::
|
|
11
|
+
|
|
12
|
+
oras://<host>[:port]/<owner>/<repo>[/<extra>]:<tag>
|
|
13
|
+
oras://<host>[:port]/<owner>/<repo>[/<extra>]@sha256:<64-hex>
|
|
14
|
+
|
|
15
|
+
Digest-pinned references skip the manifest fetch (the digest IS the
|
|
16
|
+
address); tag references go through manifest resolution + layer pick.
|
|
17
|
+
|
|
18
|
+
Why ``oras://`` and not ``ghcr:``
|
|
19
|
+
---------------------------------
|
|
20
|
+
|
|
21
|
+
The ORAS spelling disambiguates from container references. A reader
|
|
22
|
+
who sees ``ghcr.io/safl/nosi/debian-sysdev:latest`` in a docs example
|
|
23
|
+
might reach for ``docker pull`` or ``podman run`` -- which would
|
|
24
|
+
fail and leave them confused, because nosi publishes disk-image
|
|
25
|
+
artifacts, not runnable container images. ``oras://`` is the
|
|
26
|
+
ecosystem term for OCI-Registry-As-Storage; an operator googling it
|
|
27
|
+
lands at oras.land which explicitly explains "store arbitrary
|
|
28
|
+
artifacts, not just containers". The ``://`` form also composes
|
|
29
|
+
with other registries -- ``oras://quay.io/...``,
|
|
30
|
+
``oras://registry.example.com:5000/...`` -- without per-registry
|
|
31
|
+
schemes.
|
|
32
|
+
|
|
33
|
+
.. _ORAS: https://oras.land/
|
|
34
|
+
|
|
35
|
+
Auth
|
|
36
|
+
----
|
|
37
|
+
|
|
38
|
+
Spec-compliant OCI v2 registries (GHCR included) return 401 on every
|
|
39
|
+
request even for public packages. Their ``/token`` endpoint
|
|
40
|
+
mints anonymous tokens on a plain credential-less GET. So the flow
|
|
41
|
+
is: hit ``https://<host>/token``, take the returned bearer, set
|
|
42
|
+
``Authorization: Bearer`` on the manifest + blob requests. No
|
|
43
|
+
registry login, no PAT, no secrets shipped.
|
|
44
|
+
|
|
45
|
+
The token endpoint is built from the URL's host (``ghcr.io`` ->
|
|
46
|
+
``https://ghcr.io/token``), which works for GHCR and any registry
|
|
47
|
+
that follows the same convention. Registries with non-standard auth
|
|
48
|
+
flows (private registries with custom realms, e.g.) would need the
|
|
49
|
+
proper ``WWW-Authenticate`` challenge dance instead -- noted as
|
|
50
|
+
future work; not needed for the homelab / nosi use case this module
|
|
51
|
+
ships for.
|
|
52
|
+
|
|
53
|
+
Layer picker
|
|
54
|
+
------------
|
|
55
|
+
|
|
56
|
+
A nosi manifest carries two layers: the ``.img.gz`` disk image and a
|
|
57
|
+
``.sha256`` sidecar. The picker drops layers whose
|
|
58
|
+
``org.opencontainers.image.title`` annotation ends in a known sidecar
|
|
59
|
+
suffix (``.sha256``, ``.sha512``, ``.sig``, ``.asc``, ``.pem``,
|
|
60
|
+
``.cert``, ``.sbom``, ``.att``, ``.json``), then takes the largest
|
|
61
|
+
remaining layer by declared size. Manifests with no useful
|
|
62
|
+
annotations fall through to the largest layer overall -- a reasonable
|
|
63
|
+
bet that the image bytes dwarf any metadata sidecar.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
from __future__ import annotations
|
|
67
|
+
|
|
68
|
+
import json
|
|
69
|
+
import re
|
|
70
|
+
import time
|
|
71
|
+
import urllib.error
|
|
72
|
+
import urllib.parse
|
|
73
|
+
import urllib.request
|
|
74
|
+
from dataclasses import dataclass
|
|
75
|
+
from typing import Any
|
|
76
|
+
|
|
77
|
+
ORAS_SCHEME = "oras://"
|
|
78
|
+
|
|
79
|
+
# Transient HTTP statuses worth retrying: 429 (rate limit, common on
|
|
80
|
+
# GHCR / Docker Hub under load) plus the gateway/server-blip 5xx range.
|
|
81
|
+
# Everything else (401/403 auth, 404 not-found, other 4xx) is permanent
|
|
82
|
+
# and raised immediately, since retrying would just stall the caller.
|
|
83
|
+
_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
|
|
84
|
+
_RETRY_ATTEMPTS = 3
|
|
85
|
+
_RETRY_BACKOFF = 0.5 # seconds; exponential: 0.5, 1.0 between attempts
|
|
86
|
+
|
|
87
|
+
# Defensive ceiling on the metadata-fetch response body. Real-world
|
|
88
|
+
# OCI tokens are <2 KiB and manifests are <100 KiB; anything beyond
|
|
89
|
+
# this is either a misconfigured registry or a deliberately hostile
|
|
90
|
+
# response. Capping the read keeps a runaway registry from exhausting
|
|
91
|
+
# the host process's RAM on a tag resolve. Blob bytes do NOT go
|
|
92
|
+
# through this helper (they stream via the cache-host's resume loop
|
|
93
|
+
# with its own bounds).
|
|
94
|
+
_MAX_METADATA_BYTES = 10 * 1024 * 1024 # 10 MiB
|
|
95
|
+
|
|
96
|
+
# Accept type covers OCI v1 + Docker v2 manifest media types so the
|
|
97
|
+
# registry doesn't bounce us with a 406 if the package was originally
|
|
98
|
+
# pushed as a Docker manifest.
|
|
99
|
+
_MANIFEST_ACCEPT = (
|
|
100
|
+
"application/vnd.oci.image.manifest.v1+json,"
|
|
101
|
+
"application/vnd.docker.distribution.manifest.v2+json"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
# Layer titles ending in any of these are non-image sidecars (sha
|
|
105
|
+
# sums, signatures, attestations); skip them when picking the image
|
|
106
|
+
# layer so a future ``oras attach`` on the same artifact doesn't
|
|
107
|
+
# silently start being served.
|
|
108
|
+
_SIDECAR_SUFFIXES = (
|
|
109
|
+
".sha256",
|
|
110
|
+
".sha512",
|
|
111
|
+
".sig",
|
|
112
|
+
".asc",
|
|
113
|
+
".pem",
|
|
114
|
+
".cert",
|
|
115
|
+
".sbom",
|
|
116
|
+
".att",
|
|
117
|
+
".json",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# Layer mediaTypes that are NEVER disk images. A manifest whose largest
|
|
121
|
+
# layer carries one of these is a Helm chart, a Cosign signature, an
|
|
122
|
+
# in-toto attestation, a SPDX SBOM, etc. Without this filter
|
|
123
|
+
# ``pick_image_layer`` would happily pick the Helm tarball or the
|
|
124
|
+
# signature blob and downstream consumers (e.g. ``bty.flash``) would
|
|
125
|
+
# write its bytes onto an operator's target disk (tar headers /
|
|
126
|
+
# signature bytes into the MBR). An operator typing ``oras://`` at a
|
|
127
|
+
# non-image artifact deserves a clear error, not corruption.
|
|
128
|
+
_NON_IMAGE_LAYER_MEDIA_TYPES = (
|
|
129
|
+
# Helm OCI charts.
|
|
130
|
+
"application/vnd.cncf.helm.chart.v1.tar+gzip",
|
|
131
|
+
"application/vnd.cncf.helm.chart.provenance.v1.prov",
|
|
132
|
+
# Cosign signatures.
|
|
133
|
+
"application/vnd.dev.cosign.simplesigning.v1+json",
|
|
134
|
+
"application/vnd.dev.cosign.artifact.sig.v1+json",
|
|
135
|
+
# in-toto attestations.
|
|
136
|
+
"application/vnd.in-toto+json",
|
|
137
|
+
"application/vnd.dev.cosign.artifact.bundle.v1+json",
|
|
138
|
+
"application/vnd.dsse.envelope.v1+json",
|
|
139
|
+
# SBOMs.
|
|
140
|
+
"application/spdx+json",
|
|
141
|
+
"application/vnd.cyclonedx+json",
|
|
142
|
+
# OCI empty descriptor (used by artifact references with no payload).
|
|
143
|
+
"application/vnd.oci.empty.v1+json",
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _is_non_image_layer(layer: dict[str, Any]) -> bool:
|
|
148
|
+
media_type = layer.get("mediaType")
|
|
149
|
+
if not isinstance(media_type, str):
|
|
150
|
+
return False
|
|
151
|
+
return media_type in _NON_IMAGE_LAYER_MEDIA_TYPES
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class OrasError(OSError):
|
|
155
|
+
"""Raised on parse / resolution / fetch errors against an OCI registry.
|
|
156
|
+
|
|
157
|
+
Inherits from :class:`OSError` so it's caught by callers that
|
|
158
|
+
handle remote-I/O failures generically -- semantically the same
|
|
159
|
+
family as :class:`urllib.error.URLError`, which also subclasses
|
|
160
|
+
:class:`OSError`. Code paths that need to distinguish ORAS-
|
|
161
|
+
specific failures from arbitrary network errors still can:
|
|
162
|
+
:class:`OrasError` is a strict subclass."""
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class OrasRef:
|
|
167
|
+
"""Parsed ``oras://`` reference.
|
|
168
|
+
|
|
169
|
+
Exactly one of ``tag`` / ``digest`` is set. ``digest`` references
|
|
170
|
+
skip the manifest fetch (the digest is content-addressed, so the
|
|
171
|
+
blob URL is fully determined). ``tag`` references go through the
|
|
172
|
+
manifest to resolve a layer digest first.
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
host: str # e.g. "ghcr.io" or "registry.example.com:5000"
|
|
176
|
+
repository: str # e.g. "safl/nosi/debian-sysdev"
|
|
177
|
+
tag: str | None = None
|
|
178
|
+
digest: str | None = None
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def manifest_locator(self) -> str:
|
|
182
|
+
"""Value used in the ``/manifests/<X>`` URL path."""
|
|
183
|
+
if self.digest is not None:
|
|
184
|
+
return self.digest
|
|
185
|
+
assert self.tag is not None, "OrasRef must have either tag or digest set"
|
|
186
|
+
return self.tag
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
# Host: DNS hostname (or registry.example.com:5000 with optional port).
|
|
190
|
+
# Repository: lowercase alnum + ``/_.-``, must contain at least one
|
|
191
|
+
# ``/`` after the host (owner + repo). Tag: OCI tag charset (alnum +
|
|
192
|
+
# ``._-``). Digest: only sha256 today; future algorithms would need
|
|
193
|
+
# extending. Layout overall::
|
|
194
|
+
#
|
|
195
|
+
# <host>[:port]/<repo>(:<tag>|@sha256:<hex>)
|
|
196
|
+
#
|
|
197
|
+
# applied to the body after stripping the ``oras://`` scheme.
|
|
198
|
+
_REF_RE = re.compile(
|
|
199
|
+
r"^"
|
|
200
|
+
r"(?P<host>[a-zA-Z0-9][a-zA-Z0-9.-]*(?::[0-9]+)?)"
|
|
201
|
+
r"/"
|
|
202
|
+
r"(?P<repo>[a-z0-9][a-z0-9/_.-]*)"
|
|
203
|
+
r"(?:(?:@(?P<digest>sha256:[0-9a-f]{64}))"
|
|
204
|
+
r"|(?::(?P<tag>[A-Za-z0-9._-]+)))"
|
|
205
|
+
r"$"
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def parse_ref(ref: str) -> OrasRef:
|
|
210
|
+
"""Parse an ``oras://`` reference into a :class:`OrasRef`.
|
|
211
|
+
|
|
212
|
+
Accepts the two canonical forms::
|
|
213
|
+
|
|
214
|
+
oras://<host>/<owner>/<repo>[/<extra>]:<tag>
|
|
215
|
+
oras://<host>/<owner>/<repo>[/<extra>]@sha256:<64-hex>
|
|
216
|
+
|
|
217
|
+
Raises :class:`OrasError` on any malformed input. The repository
|
|
218
|
+
component must contain at least one ``/`` -- a bare top-level
|
|
219
|
+
path like ``oras://ghcr.io/nosi:latest`` is rejected because OCI's
|
|
220
|
+
URL scheme requires owner+repo under the host.
|
|
221
|
+
"""
|
|
222
|
+
if not ref.startswith(ORAS_SCHEME):
|
|
223
|
+
raise OrasError(f"not an oras:// reference: {ref!r}")
|
|
224
|
+
body = ref[len(ORAS_SCHEME) :]
|
|
225
|
+
if not body:
|
|
226
|
+
raise OrasError(f"empty oras:// reference: {ref!r}")
|
|
227
|
+
match = _REF_RE.match(body)
|
|
228
|
+
if match is None:
|
|
229
|
+
raise OrasError(
|
|
230
|
+
f"malformed oras:// reference {ref!r}: "
|
|
231
|
+
f"expected oras://<host>/<owner>/<repo>:<tag> or "
|
|
232
|
+
f"oras://<host>/<owner>/<repo>@sha256:<hex>"
|
|
233
|
+
)
|
|
234
|
+
repo = match.group("repo")
|
|
235
|
+
if "/" not in repo:
|
|
236
|
+
raise OrasError(
|
|
237
|
+
f"oras:// reference must include <host>/<owner>/<repo>: "
|
|
238
|
+
f"{ref!r} (got bare repo {repo!r} after host)"
|
|
239
|
+
)
|
|
240
|
+
return OrasRef(
|
|
241
|
+
host=match.group("host"),
|
|
242
|
+
repository=repo,
|
|
243
|
+
tag=match.group("tag"),
|
|
244
|
+
digest=match.group("digest"),
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _urlopen_retry(req: urllib.request.Request | str, *, timeout: float) -> bytes:
|
|
249
|
+
"""GET ``req`` and return the body bytes, retrying transient
|
|
250
|
+
failures with exponential backoff.
|
|
251
|
+
|
|
252
|
+
Retries on :data:`_RETRYABLE_STATUS` (429 / 5xx) and on raw
|
|
253
|
+
connection errors / timeouts. Permanent HTTP errors (401/403/404,
|
|
254
|
+
other 4xx) raise immediately -- a retry can't fix an auth or
|
|
255
|
+
not-found failure, only delay the inevitable. After
|
|
256
|
+
:data:`_RETRY_ATTEMPTS` the last error is re-raised so the caller's
|
|
257
|
+
``except`` (which wraps into :class:`OrasError`) still fires.
|
|
258
|
+
"""
|
|
259
|
+
last: BaseException | None = None
|
|
260
|
+
for attempt in range(_RETRY_ATTEMPTS):
|
|
261
|
+
try:
|
|
262
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
263
|
+
# Cap the read: a registry that returns a 500 MiB
|
|
264
|
+
# "manifest" should fail loud, not silently consume
|
|
265
|
+
# process RAM. ``read(N)`` returns up to N bytes; we
|
|
266
|
+
# take N+1 and reject the response if it overflows.
|
|
267
|
+
data = bytes(resp.read(_MAX_METADATA_BYTES + 1))
|
|
268
|
+
if len(data) > _MAX_METADATA_BYTES:
|
|
269
|
+
raise OrasError(f"oras metadata response exceeded {_MAX_METADATA_BYTES} bytes")
|
|
270
|
+
return data
|
|
271
|
+
except urllib.error.HTTPError as exc:
|
|
272
|
+
if exc.code not in _RETRYABLE_STATUS:
|
|
273
|
+
raise
|
|
274
|
+
last = exc
|
|
275
|
+
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as exc:
|
|
276
|
+
last = exc
|
|
277
|
+
if attempt < _RETRY_ATTEMPTS - 1:
|
|
278
|
+
time.sleep(_RETRY_BACKOFF * (2**attempt))
|
|
279
|
+
assert last is not None # loop ran at least once
|
|
280
|
+
raise last
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def parse_www_authenticate(header: str) -> dict[str, str]:
|
|
284
|
+
"""Parse an OCI ``WWW-Authenticate: Bearer realm="...",service="...",
|
|
285
|
+
scope="..."`` challenge into a dict of its (lower-cased) params.
|
|
286
|
+
|
|
287
|
+
Returns ``{}`` for a non-Bearer or unparseable challenge so callers
|
|
288
|
+
can treat "no usable challenge" uniformly.
|
|
289
|
+
"""
|
|
290
|
+
header = header.strip()
|
|
291
|
+
if header[:7].lower() != "bearer ":
|
|
292
|
+
return {}
|
|
293
|
+
return {m.group(1).lower(): m.group(2) for m in re.finditer(r'(\w+)="([^"]*)"', header[7:])}
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _token_from_endpoint(url: str, host: str, repository: str, *, timeout: float) -> str:
|
|
297
|
+
"""GET a token endpoint and extract the bearer. Raises OrasError."""
|
|
298
|
+
payload = json.loads(_urlopen_retry(url, timeout=timeout))
|
|
299
|
+
# OCI registries return ``token``; some spell it ``access_token``.
|
|
300
|
+
token = payload.get("token") or payload.get("access_token")
|
|
301
|
+
if not isinstance(token, str) or not token:
|
|
302
|
+
raise OrasError(
|
|
303
|
+
f"oras token response for {host}/{repository} did not contain "
|
|
304
|
+
f"a token (keys: {sorted(payload.keys())})"
|
|
305
|
+
)
|
|
306
|
+
return token
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _discover_bearer_challenge(host: str, *, timeout: float) -> dict[str, str]:
|
|
310
|
+
"""Probe ``GET https://{host}/v2/`` and return the parsed Bearer
|
|
311
|
+
challenge from the 401 response's ``WWW-Authenticate`` header.
|
|
312
|
+
|
|
313
|
+
The spec-compliant way to find a registry's token endpoint when it
|
|
314
|
+
isn't the GHCR-convention ``https://<host>/token`` (Docker Hub uses
|
|
315
|
+
``auth.docker.io``, e.g.). Returns ``{}`` if the registry answers
|
|
316
|
+
200 (no auth) or sends no usable challenge."""
|
|
317
|
+
req = urllib.request.Request(f"https://{host}/v2/", method="GET")
|
|
318
|
+
try:
|
|
319
|
+
with urllib.request.urlopen(req, timeout=timeout):
|
|
320
|
+
return {} # 200: no auth challenge to discover
|
|
321
|
+
except urllib.error.HTTPError as exc:
|
|
322
|
+
if exc.code != 401 or exc.headers is None:
|
|
323
|
+
return {}
|
|
324
|
+
return parse_www_authenticate(exc.headers.get("WWW-Authenticate", ""))
|
|
325
|
+
except (urllib.error.URLError, TimeoutError, OSError):
|
|
326
|
+
return {}
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def fetch_anonymous_token(host: str, repository: str, *, timeout: float = 30.0) -> str:
|
|
330
|
+
"""Grab an anonymous bearer token for ``repository:pull``.
|
|
331
|
+
|
|
332
|
+
Spec-compliant OCI v2 registries expose a token endpoint that mints
|
|
333
|
+
short-lived anonymous bearers for public packages (no creds, no
|
|
334
|
+
PAT). Two-step:
|
|
335
|
+
|
|
336
|
+
1. Try the GHCR / oras.land convention ``https://<host>/token`` --
|
|
337
|
+
one request for the common case.
|
|
338
|
+
2. If that fails, fall back to the spec discovery: ping
|
|
339
|
+
``GET /v2/``, read the realm advertised in the ``WWW-Authenticate``
|
|
340
|
+
Bearer challenge, and fetch from there. This covers registries
|
|
341
|
+
whose token endpoint isn't ``<host>/token`` (Docker Hub ->
|
|
342
|
+
``auth.docker.io``, Quay's custom realm, etc.).
|
|
343
|
+
"""
|
|
344
|
+
scope = f"repository:{urllib.parse.quote(repository, safe='/')}:pull"
|
|
345
|
+
conv_url = f"https://{host}/token?service={host}&scope={scope}"
|
|
346
|
+
try:
|
|
347
|
+
return _token_from_endpoint(conv_url, host, repository, timeout=timeout)
|
|
348
|
+
except (OSError, json.JSONDecodeError, ValueError, OrasError) as conv_exc:
|
|
349
|
+
# Convention failed; try spec discovery before giving up.
|
|
350
|
+
challenge = _discover_bearer_challenge(host, timeout=timeout)
|
|
351
|
+
realm = challenge.get("realm")
|
|
352
|
+
if realm:
|
|
353
|
+
service = challenge.get("service", host)
|
|
354
|
+
disc_url = f"{realm}?service={urllib.parse.quote(service, safe='')}&scope={scope}"
|
|
355
|
+
try:
|
|
356
|
+
return _token_from_endpoint(disc_url, host, repository, timeout=timeout)
|
|
357
|
+
except (OSError, json.JSONDecodeError, ValueError, OrasError) as exc:
|
|
358
|
+
raise OrasError(
|
|
359
|
+
f"oras token fetch failed for {host}/{repository} via discovered "
|
|
360
|
+
f"realm {realm}: {exc}"
|
|
361
|
+
) from exc
|
|
362
|
+
# No usable discovery; surface the original conventional error.
|
|
363
|
+
raise OrasError(
|
|
364
|
+
f"oras token fetch failed for {host}/{repository}: {conv_exc}"
|
|
365
|
+
) from conv_exc
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def fetch_manifest(ref: OrasRef, token: str, *, timeout: float = 30.0) -> dict[str, Any]:
|
|
369
|
+
"""Fetch the OCI manifest for ``ref`` using a previously-acquired token.
|
|
370
|
+
|
|
371
|
+
Returns the parsed JSON. Raises :class:`OrasError` on network or
|
|
372
|
+
parse failure. The caller is responsible for layer selection.
|
|
373
|
+
"""
|
|
374
|
+
locator = urllib.parse.quote(ref.manifest_locator, safe=":")
|
|
375
|
+
url = f"https://{ref.host}/v2/{ref.repository}/manifests/{locator}"
|
|
376
|
+
request = urllib.request.Request(
|
|
377
|
+
url,
|
|
378
|
+
headers={
|
|
379
|
+
"Authorization": f"Bearer {token}",
|
|
380
|
+
"Accept": _MANIFEST_ACCEPT,
|
|
381
|
+
},
|
|
382
|
+
)
|
|
383
|
+
try:
|
|
384
|
+
payload = json.loads(_urlopen_retry(request, timeout=timeout))
|
|
385
|
+
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
|
386
|
+
raise OrasError(
|
|
387
|
+
f"oras manifest fetch failed for "
|
|
388
|
+
f"{ref.host}/{ref.repository}:{ref.manifest_locator}: {exc}"
|
|
389
|
+
) from exc
|
|
390
|
+
if not isinstance(payload, dict):
|
|
391
|
+
raise OrasError(
|
|
392
|
+
f"oras manifest for "
|
|
393
|
+
f"{ref.host}/{ref.repository}:{ref.manifest_locator} "
|
|
394
|
+
f"is not a JSON object"
|
|
395
|
+
)
|
|
396
|
+
return payload
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _layer_title(layer: dict[str, Any]) -> str:
|
|
400
|
+
annotations = layer.get("annotations")
|
|
401
|
+
if not isinstance(annotations, dict):
|
|
402
|
+
return ""
|
|
403
|
+
title = annotations.get("org.opencontainers.image.title", "")
|
|
404
|
+
return title if isinstance(title, str) else ""
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def pick_image_layer(manifest: dict[str, Any]) -> dict[str, Any]:
|
|
408
|
+
"""Pick the disk-image layer from a (possibly multi-layer) manifest.
|
|
409
|
+
|
|
410
|
+
Drops sidecar-looking layers by title-annotation suffix, then takes
|
|
411
|
+
the largest by declared ``size``. A manifest with no usable
|
|
412
|
+
annotations falls through to the largest layer overall -- the image
|
|
413
|
+
bytes will dwarf any metadata blob in practice.
|
|
414
|
+
|
|
415
|
+
Raises :class:`OrasError` if the manifest has no layers at all.
|
|
416
|
+
"""
|
|
417
|
+
layers = manifest.get("layers")
|
|
418
|
+
if not isinstance(layers, list) or not layers:
|
|
419
|
+
# A multi-arch image *index* (``manifests`` instead of
|
|
420
|
+
# ``layers``) is a common cause: a rolling tag that resolves
|
|
421
|
+
# to an OCI index rather than a single artifact manifest. Name
|
|
422
|
+
# it so the operator points at a concrete manifest/digest
|
|
423
|
+
# instead of staring at "no layers".
|
|
424
|
+
if isinstance(manifest.get("manifests"), list) and manifest["manifests"]:
|
|
425
|
+
raise OrasError(
|
|
426
|
+
"oras ref resolved to a multi-arch image index, not a single "
|
|
427
|
+
"artifact manifest; reference a concrete platform manifest by "
|
|
428
|
+
"its @sha256:<digest> instead of the index tag"
|
|
429
|
+
)
|
|
430
|
+
raise OrasError("manifest has no layers")
|
|
431
|
+
|
|
432
|
+
# First pass: drop layers whose mediaType is definitively not a
|
|
433
|
+
# disk image (Helm chart, Cosign sig, SBOM, attestation, ...).
|
|
434
|
+
# Unlike title-suffix sidecars, these are never the right pick AND
|
|
435
|
+
# never get a fallback; flashing a Helm chart's tar+gzip into a
|
|
436
|
+
# disk's MBR is the data-loss scenario this guard exists for.
|
|
437
|
+
rejected_media_types: list[str] = []
|
|
438
|
+
image_candidates: list[dict[str, Any]] = []
|
|
439
|
+
for layer in layers:
|
|
440
|
+
if not isinstance(layer, dict):
|
|
441
|
+
continue
|
|
442
|
+
if _is_non_image_layer(layer):
|
|
443
|
+
mt = layer.get("mediaType")
|
|
444
|
+
if isinstance(mt, str):
|
|
445
|
+
rejected_media_types.append(mt)
|
|
446
|
+
continue
|
|
447
|
+
image_candidates.append(layer)
|
|
448
|
+
if not image_candidates:
|
|
449
|
+
if rejected_media_types:
|
|
450
|
+
unique = ", ".join(sorted(set(rejected_media_types)))
|
|
451
|
+
raise OrasError(
|
|
452
|
+
f"oras ref resolved to a non-disk-image artifact "
|
|
453
|
+
f"(layer mediaTypes: {unique}); refusing to serve "
|
|
454
|
+
f"signature / Helm chart / SBOM blobs as disk images"
|
|
455
|
+
)
|
|
456
|
+
raise OrasError("manifest has no usable layers")
|
|
457
|
+
|
|
458
|
+
# Second pass: among the image-eligible layers, prefer ones that
|
|
459
|
+
# DON'T look like title-annotation sidecars (.sha256 / .sig / ...
|
|
460
|
+
# filenames). If every remaining layer carries a sidecar-looking
|
|
461
|
+
# title, fall back to "pick the largest" so the resolver gets a
|
|
462
|
+
# chance to fail loudly downstream rather than this picker
|
|
463
|
+
# mis-classifying.
|
|
464
|
+
image_like: list[dict[str, Any]] = []
|
|
465
|
+
for layer in image_candidates:
|
|
466
|
+
title = _layer_title(layer)
|
|
467
|
+
if title and any(title.endswith(suffix) for suffix in _SIDECAR_SUFFIXES):
|
|
468
|
+
continue
|
|
469
|
+
image_like.append(layer)
|
|
470
|
+
candidates = image_like or image_candidates
|
|
471
|
+
return max(candidates, key=lambda layer: layer.get("size") or 0)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
@dataclass(frozen=True)
|
|
475
|
+
class ResolvedBlob:
|
|
476
|
+
"""Everything a fetcher needs to stream the image bytes.
|
|
477
|
+
|
|
478
|
+
``blob_url`` is the final ``/v2/<repo>/blobs/sha256:<digest>``
|
|
479
|
+
endpoint. ``headers`` carries the bearer the registry expects. The
|
|
480
|
+
``digest`` is what the fetcher should verify the downloaded bytes
|
|
481
|
+
against -- when the caller started from a tag, this is the digest
|
|
482
|
+
the registry resolved to right now, frozen for the rest of the
|
|
483
|
+
fetch.
|
|
484
|
+
"""
|
|
485
|
+
|
|
486
|
+
blob_url: str
|
|
487
|
+
headers: dict[str, str]
|
|
488
|
+
digest: str
|
|
489
|
+
size: int | None
|
|
490
|
+
title: str | None
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def resolve_ref(ref: str | OrasRef, *, timeout: float = 30.0) -> ResolvedBlob:
|
|
494
|
+
"""Resolve an ``oras://`` reference (or pre-parsed :class:`OrasRef`)
|
|
495
|
+
to a :class:`ResolvedBlob`.
|
|
496
|
+
|
|
497
|
+
For tag references: anonymous token -> manifest -> layer pick ->
|
|
498
|
+
``ResolvedBlob`` with the layer's content-addressed digest. The
|
|
499
|
+
digest is frozen at resolve time so a tag that moves under us
|
|
500
|
+
between resolve and fetch still produces the bytes we committed to.
|
|
501
|
+
|
|
502
|
+
For digest-pinned references: anonymous token only; the blob URL is
|
|
503
|
+
fully determined by the digest, the manifest fetch is skipped, and
|
|
504
|
+
size / title come back as ``None`` (the descriptor's optional
|
|
505
|
+
``size_bytes`` field can carry that info instead if known).
|
|
506
|
+
"""
|
|
507
|
+
if isinstance(ref, str):
|
|
508
|
+
ref = parse_ref(ref)
|
|
509
|
+
token = fetch_anonymous_token(ref.host, ref.repository, timeout=timeout)
|
|
510
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
511
|
+
|
|
512
|
+
if ref.digest is not None:
|
|
513
|
+
digest = ref.digest
|
|
514
|
+
size: int | None = None
|
|
515
|
+
title: str | None = None
|
|
516
|
+
else:
|
|
517
|
+
manifest = fetch_manifest(ref, token, timeout=timeout)
|
|
518
|
+
layer = pick_image_layer(manifest)
|
|
519
|
+
raw_digest = layer.get("digest")
|
|
520
|
+
if not isinstance(raw_digest, str) or not raw_digest.startswith("sha256:"):
|
|
521
|
+
raise OrasError(
|
|
522
|
+
f"picked layer for "
|
|
523
|
+
f"{ref.host}/{ref.repository}:{ref.manifest_locator} "
|
|
524
|
+
f"has unusable digest {raw_digest!r}"
|
|
525
|
+
)
|
|
526
|
+
digest = raw_digest
|
|
527
|
+
layer_size = layer.get("size")
|
|
528
|
+
size = layer_size if isinstance(layer_size, int) else None
|
|
529
|
+
title = _layer_title(layer) or None
|
|
530
|
+
|
|
531
|
+
blob_url = f"https://{ref.host}/v2/{ref.repository}/blobs/{digest}"
|
|
532
|
+
return ResolvedBlob(
|
|
533
|
+
blob_url=blob_url,
|
|
534
|
+
headers=headers,
|
|
535
|
+
digest=digest,
|
|
536
|
+
size=size,
|
|
537
|
+
title=title,
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def is_oras_url(url: str) -> bool:
|
|
542
|
+
"""True iff ``url`` is an ``oras://`` reference rather than http(s)://."""
|
|
543
|
+
return url.startswith(ORAS_SCHEME)
|