oic-sync 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oic_sync/__init__.py +5 -0
- oic_sync/__main__.py +3 -0
- oic_sync/backends/__init__.py +5 -0
- oic_sync/backends/base.py +76 -0
- oic_sync/backends/openstack.py +461 -0
- oic_sync/backends/proxmox.py +639 -0
- oic_sync/cli.py +480 -0
- oic_sync/engine.py +331 -0
- oic_sync/hashing.py +82 -0
- oic_sync/http.py +27 -0
- oic_sync/index.py +287 -0
- oic_sync/signature.py +115 -0
- oic_sync-0.1.0.dist-info/METADATA +331 -0
- oic_sync-0.1.0.dist-info/RECORD +17 -0
- oic_sync-0.1.0.dist-info/WHEEL +4 -0
- oic_sync-0.1.0.dist-info/entry_points.txt +2 -0
- oic_sync-0.1.0.dist-info/licenses/LICENSE +202 -0
oic_sync/index.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""The openimages.cloud catalog index (schema oic-index/v1).
|
|
2
|
+
|
|
3
|
+
The index is generated by the website build (oic-website, src/lib/catalog-index.ts) and lists,
|
|
4
|
+
for every image, the artifacts of its latest version with digests, sizes, the cosign bundle and
|
|
5
|
+
the complete Glance property set. The schema only gains fields over time; a breaking change
|
|
6
|
+
would bump it to oic-index/v2, which this version refuses.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Iterable, Mapping
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
DEFAULT_INDEX_URL = "https://images.openimages.cloud/index.json"
|
|
20
|
+
SCHEMA = "oic-index/v1"
|
|
21
|
+
|
|
22
|
+
# Service images (Octavia amphora, Manila service VM) are owned by an OpenStack service and
|
|
23
|
+
# imported with that service's procedure. They are only synced when named with --image.
|
|
24
|
+
SERVICE_IMAGES = frozenset(
|
|
25
|
+
{"octavia-amphora", "manila-service-image", "trove-guest-image", "cirros"}
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
# Index size cap: the real index is around 60 KiB.
|
|
29
|
+
MAX_INDEX_BYTES = 16 * 1024 * 1024
|
|
30
|
+
|
|
31
|
+
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
|
32
|
+
_HEX128 = re.compile(r"^[0-9a-f]{128}$")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class IndexFormatError(ValueError):
|
|
36
|
+
"""The index is unreachable, malformed or of an unsupported schema."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Signing:
|
|
41
|
+
"""Expected cosign keyless signer, as published in the index."""
|
|
42
|
+
|
|
43
|
+
identity_regexp: str
|
|
44
|
+
oidc_issuer: str
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class Artifact:
|
|
49
|
+
variant: str | None
|
|
50
|
+
version: str
|
|
51
|
+
arch: str
|
|
52
|
+
format: str
|
|
53
|
+
firmware: str
|
|
54
|
+
url: str
|
|
55
|
+
sha256: str
|
|
56
|
+
sha512: str | None
|
|
57
|
+
size: int | None
|
|
58
|
+
virtual_size: int | None
|
|
59
|
+
bundle_url: str | None
|
|
60
|
+
manifest_url: str | None
|
|
61
|
+
signed: bool
|
|
62
|
+
built_at: str | None
|
|
63
|
+
glance_properties: Mapping[str, str] = field(default_factory=dict)
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def min_disk_gib(self) -> int | None:
|
|
67
|
+
"""Smallest root disk (GiB, rounded up) that fits the image."""
|
|
68
|
+
if not self.virtual_size:
|
|
69
|
+
return None
|
|
70
|
+
return math.ceil(self.virtual_size / 2**30)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class Image:
|
|
75
|
+
id: str
|
|
76
|
+
title: str
|
|
77
|
+
status: str
|
|
78
|
+
os_distro: str | None
|
|
79
|
+
os_version: str | None
|
|
80
|
+
login_user: str | None
|
|
81
|
+
latest_version: str
|
|
82
|
+
optional_glance_properties: Mapping[str, str]
|
|
83
|
+
artifacts: tuple[Artifact, ...]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class CatalogIndex:
|
|
88
|
+
schema: str
|
|
89
|
+
generated_at: str
|
|
90
|
+
base_url: str
|
|
91
|
+
signing: Signing
|
|
92
|
+
images: tuple[Image, ...]
|
|
93
|
+
|
|
94
|
+
def image(self, image_id: str) -> Image | None:
|
|
95
|
+
return next((i for i in self.images if i.id == image_id), None)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class Target:
|
|
100
|
+
"""One artifact to keep in sync: an image of the index, in one variant."""
|
|
101
|
+
|
|
102
|
+
image: Image
|
|
103
|
+
artifact: Artifact
|
|
104
|
+
stable_name: str
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def label(self) -> str:
|
|
108
|
+
"""Short human label: debian-13, alpine-linux/uefi."""
|
|
109
|
+
if self.artifact.variant:
|
|
110
|
+
return f"{self.image.id}/{self.artifact.variant}"
|
|
111
|
+
return self.image.id
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _str(d: Mapping[str, Any], key: str, where: str, *, optional: bool = False) -> str | None:
|
|
115
|
+
value = d.get(key)
|
|
116
|
+
if value is None and optional:
|
|
117
|
+
return None
|
|
118
|
+
if not isinstance(value, str) or not value:
|
|
119
|
+
raise IndexFormatError(f"{where}: missing or invalid '{key}'")
|
|
120
|
+
return value
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _int(d: Mapping[str, Any], key: str) -> int | None:
|
|
124
|
+
value = d.get(key)
|
|
125
|
+
if value is None:
|
|
126
|
+
return None
|
|
127
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
128
|
+
raise IndexFormatError(f"invalid '{key}': {value!r}")
|
|
129
|
+
return value
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _props(value: Any, where: str) -> dict[str, str]:
|
|
133
|
+
if value is None:
|
|
134
|
+
return {}
|
|
135
|
+
if not isinstance(value, dict):
|
|
136
|
+
raise IndexFormatError(f"{where}: glance properties must be an object")
|
|
137
|
+
return {str(k): str(v) for k, v in value.items()}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _parse_artifact(raw: Any, where: str) -> Artifact:
|
|
141
|
+
if not isinstance(raw, dict):
|
|
142
|
+
raise IndexFormatError(f"{where}: artifact must be an object")
|
|
143
|
+
sha256 = (_str(raw, "sha256", where) or "").lower()
|
|
144
|
+
if not _HEX64.match(sha256):
|
|
145
|
+
raise IndexFormatError(f"{where}: sha256 is not 64 hex characters")
|
|
146
|
+
sha512 = raw.get("sha512")
|
|
147
|
+
if sha512 is not None:
|
|
148
|
+
sha512 = str(sha512).lower()
|
|
149
|
+
if not _HEX128.match(sha512):
|
|
150
|
+
raise IndexFormatError(f"{where}: sha512 is not 128 hex characters")
|
|
151
|
+
variant = raw.get("variant")
|
|
152
|
+
if variant is not None and (not isinstance(variant, str) or not variant):
|
|
153
|
+
raise IndexFormatError(f"{where}: invalid variant")
|
|
154
|
+
url = _str(raw, "url", where)
|
|
155
|
+
assert url is not None
|
|
156
|
+
if not url.startswith("https://"):
|
|
157
|
+
raise IndexFormatError(f"{where}: artifact url must be https")
|
|
158
|
+
return Artifact(
|
|
159
|
+
variant=variant,
|
|
160
|
+
version=_str(raw, "version", where) or "",
|
|
161
|
+
arch=_str(raw, "arch", where, optional=True) or "x86_64",
|
|
162
|
+
format=_str(raw, "format", where, optional=True) or "qcow2",
|
|
163
|
+
firmware=_str(raw, "firmware", where, optional=True) or "bios",
|
|
164
|
+
url=url,
|
|
165
|
+
sha256=sha256,
|
|
166
|
+
sha512=sha512,
|
|
167
|
+
size=_int(raw, "size"),
|
|
168
|
+
virtual_size=_int(raw, "virtual_size"),
|
|
169
|
+
bundle_url=_str(raw, "bundle_url", where, optional=True),
|
|
170
|
+
manifest_url=_str(raw, "manifest_url", where, optional=True),
|
|
171
|
+
signed=bool(raw.get("signed", False)),
|
|
172
|
+
built_at=_str(raw, "built_at", where, optional=True),
|
|
173
|
+
glance_properties=_props(raw.get("glance_properties"), where),
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def parse_index(data: Any) -> CatalogIndex:
|
|
178
|
+
"""Validate and convert the decoded index.json."""
|
|
179
|
+
if not isinstance(data, dict):
|
|
180
|
+
raise IndexFormatError("index is not a JSON object")
|
|
181
|
+
schema = data.get("schema")
|
|
182
|
+
if schema != SCHEMA:
|
|
183
|
+
raise IndexFormatError(f"unsupported index schema {schema!r} (expected {SCHEMA})")
|
|
184
|
+
signing_raw = data.get("signing")
|
|
185
|
+
if not isinstance(signing_raw, dict):
|
|
186
|
+
raise IndexFormatError("index has no 'signing' block")
|
|
187
|
+
if signing_raw.get("type") != "cosign-keyless":
|
|
188
|
+
raise IndexFormatError(f"unsupported signing type {signing_raw.get('type')!r}")
|
|
189
|
+
signing = Signing(
|
|
190
|
+
identity_regexp=_str(signing_raw, "certificate_identity_regexp", "signing") or "",
|
|
191
|
+
oidc_issuer=_str(signing_raw, "certificate_oidc_issuer", "signing") or "",
|
|
192
|
+
)
|
|
193
|
+
try:
|
|
194
|
+
re.compile(signing.identity_regexp)
|
|
195
|
+
except re.error as e:
|
|
196
|
+
raise IndexFormatError(f"signing: invalid identity regexp: {e}") from e
|
|
197
|
+
|
|
198
|
+
images_raw = data.get("images")
|
|
199
|
+
if not isinstance(images_raw, list):
|
|
200
|
+
raise IndexFormatError("index has no 'images' list")
|
|
201
|
+
images = []
|
|
202
|
+
for raw in images_raw:
|
|
203
|
+
if not isinstance(raw, dict):
|
|
204
|
+
raise IndexFormatError("image entry must be an object")
|
|
205
|
+
image_id = _str(raw, "id", "image") or ""
|
|
206
|
+
artifacts_raw = raw.get("artifacts") or []
|
|
207
|
+
if not isinstance(artifacts_raw, list):
|
|
208
|
+
raise IndexFormatError(f"{image_id}: artifacts must be a list")
|
|
209
|
+
artifacts = tuple(
|
|
210
|
+
_parse_artifact(a, f"{image_id}[{n}]") for n, a in enumerate(artifacts_raw)
|
|
211
|
+
)
|
|
212
|
+
images.append(
|
|
213
|
+
Image(
|
|
214
|
+
id=image_id,
|
|
215
|
+
title=_str(raw, "title", image_id) or image_id,
|
|
216
|
+
status=_str(raw, "status", image_id, optional=True) or "stable",
|
|
217
|
+
os_distro=_str(raw, "os_distro", image_id, optional=True),
|
|
218
|
+
os_version=_str(raw, "os_version", image_id, optional=True),
|
|
219
|
+
login_user=_str(raw, "login_user", image_id, optional=True),
|
|
220
|
+
latest_version=_str(raw, "latest_version", image_id, optional=True) or "",
|
|
221
|
+
optional_glance_properties=_props(raw.get("optional_glance_properties"), image_id),
|
|
222
|
+
artifacts=artifacts,
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
return CatalogIndex(
|
|
226
|
+
schema=schema,
|
|
227
|
+
generated_at=str(data.get("generated_at") or ""),
|
|
228
|
+
base_url=str(data.get("base_url") or ""),
|
|
229
|
+
signing=signing,
|
|
230
|
+
images=tuple(images),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def fetch_index(client: httpx.Client, url: str = DEFAULT_INDEX_URL) -> CatalogIndex:
|
|
235
|
+
"""Download and parse the index."""
|
|
236
|
+
try:
|
|
237
|
+
response = client.get(url)
|
|
238
|
+
response.raise_for_status()
|
|
239
|
+
except httpx.HTTPError as e:
|
|
240
|
+
raise IndexFormatError(f"cannot fetch index {url}: {e}") from e
|
|
241
|
+
if len(response.content) > MAX_INDEX_BYTES:
|
|
242
|
+
raise IndexFormatError(f"index {url} is larger than {MAX_INDEX_BYTES} bytes")
|
|
243
|
+
try:
|
|
244
|
+
data = response.json()
|
|
245
|
+
except ValueError as e:
|
|
246
|
+
raise IndexFormatError(f"index {url} is not valid JSON: {e}") from e
|
|
247
|
+
return parse_index(data)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def stable_name(image: Image, artifact: Artifact) -> str:
|
|
251
|
+
"""Name held by the newest version: the image title, plus the variant when there are several.
|
|
252
|
+
|
|
253
|
+
Matches the osism export of the website: "Alpine Linux (UEFI)", "Debian 13".
|
|
254
|
+
"""
|
|
255
|
+
if artifact.variant and len(image.artifacts) > 1:
|
|
256
|
+
return f"{image.title} ({artifact.variant.upper()})"
|
|
257
|
+
return image.title
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def select_targets(
|
|
261
|
+
index: CatalogIndex,
|
|
262
|
+
images: Iterable[str] | None = None,
|
|
263
|
+
variants: Iterable[str] | None = None,
|
|
264
|
+
*,
|
|
265
|
+
include_service: bool = False,
|
|
266
|
+
) -> list[Target]:
|
|
267
|
+
"""Artifacts to sync.
|
|
268
|
+
|
|
269
|
+
images: ids to keep (all non-service images when empty); an unknown id is an error.
|
|
270
|
+
variants: variant names to keep; artifacts without a variant are never filtered out.
|
|
271
|
+
"""
|
|
272
|
+
wanted = list(dict.fromkeys(images or []))
|
|
273
|
+
unknown = [i for i in wanted if index.image(i) is None]
|
|
274
|
+
if unknown:
|
|
275
|
+
raise IndexFormatError(f"unknown image id(s) in the index: {', '.join(unknown)}")
|
|
276
|
+
variant_set = set(variants or [])
|
|
277
|
+
targets: list[Target] = []
|
|
278
|
+
for image in index.images:
|
|
279
|
+
if wanted and image.id not in wanted:
|
|
280
|
+
continue
|
|
281
|
+
if not wanted and image.id in SERVICE_IMAGES and not include_service:
|
|
282
|
+
continue
|
|
283
|
+
for artifact in image.artifacts:
|
|
284
|
+
if variant_set and artifact.variant and artifact.variant not in variant_set:
|
|
285
|
+
continue
|
|
286
|
+
targets.append(Target(image, artifact, stable_name(image, artifact)))
|
|
287
|
+
return targets
|
oic_sync/signature.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Cosign keyless signature verification of index artifacts.
|
|
2
|
+
|
|
3
|
+
Every artifact is signed by the open-img-cloud build workflows with cosign keyless (Sigstore).
|
|
4
|
+
The bundle signs the sha256 of the qcow2, so it is verified against the digest published in the
|
|
5
|
+
index, without downloading the image: the platform later checks that what it imports matches
|
|
6
|
+
that digest. The expected identity (a regexp on the workflow URI) and OIDC issuer come from
|
|
7
|
+
the index `signing` block.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
from typing import Any, Protocol
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from oic_sync.index import Artifact, Signing
|
|
18
|
+
|
|
19
|
+
MAX_BUNDLE_BYTES = 1024 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SignatureError(Exception):
|
|
23
|
+
"""The artifact signature is missing or does not verify."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _Verifier(Protocol):
|
|
27
|
+
def verify_artifact(self, input_: Any, bundle: Any, policy: Any) -> None: ...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RegexpIdentityPolicy:
|
|
31
|
+
"""sigstore verification policy: OIDC issuer equality and SAN URI regexp search.
|
|
32
|
+
|
|
33
|
+
Same semantics as `cosign verify-blob --certificate-identity-regexp
|
|
34
|
+
--certificate-oidc-issuer` (Go regexp search, unanchored unless the pattern anchors it).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self, identity_regexp: str, oidc_issuer: str) -> None:
|
|
38
|
+
self._identity = re.compile(identity_regexp)
|
|
39
|
+
self._issuer = oidc_issuer
|
|
40
|
+
self.matched_identity: str | None = None
|
|
41
|
+
|
|
42
|
+
def verify(self, cert: Any) -> None:
|
|
43
|
+
from sigstore.verify import policy
|
|
44
|
+
|
|
45
|
+
# Fulcio writes the issuer in the V2 extension (DER UTF8String) and, for older
|
|
46
|
+
# certificates, in the legacy V1 extension: accept either.
|
|
47
|
+
policy.AnyOf([policy.OIDCIssuerV2(self._issuer), policy.OIDCIssuer(self._issuer)]).verify(
|
|
48
|
+
cert
|
|
49
|
+
)
|
|
50
|
+
try:
|
|
51
|
+
san = cert.extensions.get_extension_for_class(policy.SubjectAlternativeName).value
|
|
52
|
+
except policy.ExtensionNotFound as e:
|
|
53
|
+
raise policy.VerificationError("certificate has no SubjectAlternativeName") from e
|
|
54
|
+
uris = san.get_values_for_type(policy.UniformResourceIdentifier)
|
|
55
|
+
for uri in uris:
|
|
56
|
+
if self._identity.search(uri):
|
|
57
|
+
self.matched_identity = uri
|
|
58
|
+
return
|
|
59
|
+
raise policy.VerificationError(
|
|
60
|
+
f"certificate identity {uris} does not match {self._identity.pattern!r}"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SignatureVerifier:
|
|
65
|
+
"""Fetches the bundle of an artifact and verifies it against the index sha256."""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self, client: httpx.Client, signing: Signing, verifier: _Verifier | None = None
|
|
69
|
+
) -> None:
|
|
70
|
+
self._client = client
|
|
71
|
+
self._signing = signing
|
|
72
|
+
self._verifier = verifier
|
|
73
|
+
|
|
74
|
+
def _get_verifier(self) -> _Verifier:
|
|
75
|
+
if self._verifier is None:
|
|
76
|
+
from sigstore.verify import Verifier
|
|
77
|
+
|
|
78
|
+
# Fetches the Sigstore public-good trust root through TUF (cached in ~/.cache).
|
|
79
|
+
self._verifier = Verifier.production()
|
|
80
|
+
return self._verifier
|
|
81
|
+
|
|
82
|
+
def _fetch_bundle(self, url: str) -> bytes:
|
|
83
|
+
try:
|
|
84
|
+
response = self._client.get(url)
|
|
85
|
+
response.raise_for_status()
|
|
86
|
+
except httpx.HTTPError as e:
|
|
87
|
+
raise SignatureError(f"cannot fetch bundle {url}: {e}") from e
|
|
88
|
+
if len(response.content) > MAX_BUNDLE_BYTES:
|
|
89
|
+
raise SignatureError(f"bundle {url} is unexpectedly large")
|
|
90
|
+
return response.content
|
|
91
|
+
|
|
92
|
+
def verify(self, artifact: Artifact) -> str:
|
|
93
|
+
"""Verify the artifact signature; return the signer identity. Raise SignatureError."""
|
|
94
|
+
if not artifact.bundle_url:
|
|
95
|
+
raise SignatureError("the index lists no cosign bundle for this artifact")
|
|
96
|
+
raw = self._fetch_bundle(artifact.bundle_url)
|
|
97
|
+
|
|
98
|
+
from sigstore.errors import Error as SigstoreError
|
|
99
|
+
from sigstore.hashes import Hashed
|
|
100
|
+
from sigstore.models import Bundle
|
|
101
|
+
from sigstore_models.common.v1 import HashAlgorithm
|
|
102
|
+
|
|
103
|
+
try:
|
|
104
|
+
bundle = Bundle.from_json(raw)
|
|
105
|
+
except Exception as e:
|
|
106
|
+
raise SignatureError(f"invalid bundle {artifact.bundle_url}: {e}") from e
|
|
107
|
+
policy = RegexpIdentityPolicy(self._signing.identity_regexp, self._signing.oidc_issuer)
|
|
108
|
+
hashed = Hashed(algorithm=HashAlgorithm.SHA2_256, digest=bytes.fromhex(artifact.sha256))
|
|
109
|
+
try:
|
|
110
|
+
self._get_verifier().verify_artifact(hashed, bundle, policy)
|
|
111
|
+
except SigstoreError as e:
|
|
112
|
+
raise SignatureError(f"signature verification failed: {e}") from e
|
|
113
|
+
except Exception as e: # fail closed on anything unexpected (trust root, parsing)
|
|
114
|
+
raise SignatureError(f"signature verification error: {e!r}") from e
|
|
115
|
+
return policy.matched_identity or "unknown"
|