oic-sync 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oic_sync/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """oic-sync: import openimages.cloud signed cloud images into cloud platforms."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ USER_AGENT = f"oic-sync/{__version__}"
oic_sync/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from oic_sync.cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,5 @@
1
+ """Platform backends: OpenStack (Glance) and Proxmox VE (VM templates, through pvesh)."""
2
+
3
+ from oic_sync.backends.base import Backend, BackendError, ManagedImage
4
+
5
+ __all__ = ["Backend", "BackendError", "ManagedImage"]
@@ -0,0 +1,76 @@
1
+ """Platform-neutral view of the images oic-sync manages, and the backend interface.
2
+
3
+ A backend (OpenStack, Proxmox VE) only has to list the images it holds for a target,
4
+ import a new one, rename/hide one, tell whether one is in use and delete one. All decisions
5
+ (what to import, which version holds the stable name, what to retire or delete) are taken by
6
+ oic_sync.engine, identically for every platform. A backend whose platform restricts names
7
+ (Proxmox VE VM names are DNS names) exposes a `naming` attribute (oic_sync.engine.Naming).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass, field
13
+ from datetime import datetime
14
+ from typing import Protocol
15
+
16
+ from oic_sync.index import Target
17
+
18
+ # Usable images: may hold the stable name or be kept as an older version.
19
+ USABLE = frozenset({"active", "deactivated"})
20
+ # Still being created or imported.
21
+ TRANSIENT = frozenset({"queued", "saving", "importing", "uploading"})
22
+ # Dead: the import failed and the image can never become usable.
23
+ FAILED = frozenset({"killed"})
24
+
25
+
26
+ class BackendError(Exception):
27
+ """A platform call failed."""
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class ManagedImage:
32
+ """An image of the platform recognised as one of ours (oic_id/oic_variant match)."""
33
+
34
+ id: str
35
+ name: str
36
+ version: str
37
+ sha256: str
38
+ status: str
39
+ hidden: bool
40
+ created_at: datetime
41
+ visibility: str = "private"
42
+ extra: dict[str, str] = field(default_factory=dict, compare=False)
43
+
44
+ @property
45
+ def usable(self) -> bool:
46
+ return self.status in USABLE
47
+
48
+
49
+ class Backend(Protocol):
50
+ """What the engine needs from a platform."""
51
+
52
+ def describe(self) -> str:
53
+ """One line naming the platform (cloud, region, project) for logs."""
54
+ ...
55
+
56
+ def list_managed(self, target: Target) -> list[ManagedImage]:
57
+ """Every image of the platform (owned by us) for this image id and variant."""
58
+ ...
59
+
60
+ def import_target(self, target: Target, name: str) -> ManagedImage:
61
+ """Create the image hidden, under `name`, import and verify it.
62
+
63
+ Returns the usable image. On any failure, the half-created image is removed and
64
+ BackendError (or HashMismatch) is raised.
65
+ """
66
+ ...
67
+
68
+ def rename(self, image: ManagedImage, name: str, hidden: bool) -> None:
69
+ """Set the name and the hidden flag of an image."""
70
+ ...
71
+
72
+ def in_use(self, image: ManagedImage) -> bool | None:
73
+ """Whether an instance or volume uses the image; None when it cannot be told."""
74
+ ...
75
+
76
+ def delete(self, image: ManagedImage) -> None: ...
@@ -0,0 +1,461 @@
1
+ """OpenStack backend: Glance v2 images, Nova and Cinder for usage checks.
2
+
3
+ Calls go through the keystoneauth adapters of an openstacksdk Connection (`conn.image`,
4
+ `conn.compute`, `conn.block_storage`) with plain REST paths, which keeps this module small
5
+ and easy to test with fakes. Authentication and endpoints come from clouds.yaml.
6
+
7
+ Our images are recognised by their properties: oic_id (+ oic_variant for multi-variant
8
+ images) and owner = current project. oic_version and oic_sha256 tell which version an image
9
+ holds.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import logging
16
+ import time
17
+ from collections.abc import Callable, Iterator, Mapping
18
+ from datetime import datetime, timezone
19
+ from typing import Any, Protocol
20
+
21
+ import httpx
22
+
23
+ from oic_sync.backends.base import BackendError, ManagedImage
24
+ from oic_sync.hashing import HashingReader, HashMismatch
25
+ from oic_sync.index import Target
26
+
27
+ log = logging.getLogger(__name__)
28
+
29
+ WEB_DOWNLOAD = "web-download"
30
+ GLANCE_DIRECT = "glance-direct"
31
+ UPLOAD = "upload"
32
+ AUTO = "auto"
33
+ IMPORT_METHODS = (AUTO, WEB_DOWNLOAD, GLANCE_DIRECT, UPLOAD)
34
+
35
+ VISIBILITIES = ("private", "shared", "community", "public")
36
+
37
+ JSON_PATCH = "application/openstack-images-v2.1-json-patch"
38
+ CHUNK = 1024 * 1024
39
+ PAGE = 200
40
+
41
+ # Keys oic-sync owns: never overridable with --property.
42
+ RESERVED_PROPERTIES = frozenset({"oic_id", "oic_variant", "oic_version", "oic_sha256"})
43
+
44
+
45
+ class RestAdapter(Protocol):
46
+ """The subset of a keystoneauth Adapter / openstacksdk Proxy used here."""
47
+
48
+ def get(self, url: str, **kwargs: Any) -> Any: ...
49
+ def post(self, url: str, **kwargs: Any) -> Any: ...
50
+ def put(self, url: str, **kwargs: Any) -> Any: ...
51
+ def patch(self, url: str, **kwargs: Any) -> Any: ...
52
+ def delete(self, url: str, **kwargs: Any) -> Any: ...
53
+
54
+
55
+ def _check(response: Any, what: str, ok: tuple[int, ...] = (200, 201, 202, 204)) -> Any:
56
+ if response.status_code not in ok:
57
+ body = (getattr(response, "text", "") or "")[:300]
58
+ raise BackendError(f"{what}: HTTP {response.status_code} {body}".rstrip())
59
+ return response
60
+
61
+
62
+ def _parse_time(value: str | None) -> datetime:
63
+ if not value:
64
+ return datetime.fromtimestamp(0, timezone.utc)
65
+ value = value.replace("Z", "+00:00")
66
+ parsed = datetime.fromisoformat(value)
67
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
68
+
69
+
70
+ class Glance:
71
+ """Thin Glance v2 client."""
72
+
73
+ def __init__(self, api: RestAdapter, project_id: str) -> None:
74
+ self.api = api
75
+ self.project_id = project_id
76
+
77
+ def import_methods(self) -> list[str]:
78
+ response = self.api.get("/info/import")
79
+ if response.status_code == 404: # interoperable import disabled or very old Glance
80
+ return []
81
+ _check(response, "GET /info/import")
82
+ return list(response.json().get("import-methods", {}).get("value", []))
83
+
84
+ def list_images(self, filters: Mapping[str, str]) -> list[dict[str, Any]]:
85
+ images: list[dict[str, Any]] = []
86
+ params: dict[str, Any] = {**filters, "limit": PAGE}
87
+ while True:
88
+ body = _check(self.api.get("/images", params=params), "GET /images").json()
89
+ page = body.get("images", [])
90
+ images.extend(page)
91
+ if not page or not body.get("next"):
92
+ return images
93
+ params = {**params, "marker": page[-1]["id"]}
94
+
95
+ def get_image(self, image_id: str) -> dict[str, Any]:
96
+ return _check(self.api.get(f"/images/{image_id}"), f"GET image {image_id}").json()
97
+
98
+ def create_image(self, body: Mapping[str, Any]) -> dict[str, Any]:
99
+ return _check(self.api.post("/images", json=dict(body)), "POST /images").json()
100
+
101
+ def update_image(self, image_id: str, ops: list[dict[str, Any]]) -> None:
102
+ _check(
103
+ self.api.patch(
104
+ f"/images/{image_id}",
105
+ data=json.dumps(ops),
106
+ headers={"Content-Type": JSON_PATCH},
107
+ ),
108
+ f"PATCH image {image_id}",
109
+ )
110
+
111
+ def upload(self, image_id: str, data: Any) -> None:
112
+ _check(
113
+ self.api.put(
114
+ f"/images/{image_id}/file",
115
+ data=data,
116
+ headers={"Content-Type": "application/octet-stream"},
117
+ ),
118
+ f"PUT image {image_id} data",
119
+ )
120
+
121
+ def stage(self, image_id: str, data: Any) -> None:
122
+ _check(
123
+ self.api.put(
124
+ f"/images/{image_id}/stage",
125
+ data=data,
126
+ headers={"Content-Type": "application/octet-stream"},
127
+ ),
128
+ f"PUT image {image_id} stage",
129
+ )
130
+
131
+ def import_image(self, image_id: str, method: str, uri: str | None = None) -> None:
132
+ body: dict[str, Any] = {"method": {"name": method}}
133
+ if uri:
134
+ body["method"]["uri"] = uri
135
+ _check(self.api.post(f"/images/{image_id}/import", json=body), f"import {image_id}")
136
+
137
+ def delete_image(self, image_id: str) -> None:
138
+ _check(self.api.delete(f"/images/{image_id}"), f"DELETE image {image_id}", (204, 404))
139
+
140
+
141
+ class OpenStackBackend:
142
+ def __init__(
143
+ self,
144
+ glance: Glance,
145
+ http: httpx.Client,
146
+ *,
147
+ compute: RestAdapter | None = None,
148
+ volume: RestAdapter | None = None,
149
+ visibility: str = "private",
150
+ import_method: str = AUTO,
151
+ extra_properties: Mapping[str, str] | None = None,
152
+ optional_properties: bool = False,
153
+ import_timeout: float = 3600.0,
154
+ poll_interval: float = 10.0,
155
+ sleep: Callable[[float], None] = time.sleep,
156
+ clock: Callable[[], float] = time.monotonic,
157
+ description: str = "openstack",
158
+ ) -> None:
159
+ if visibility not in VISIBILITIES:
160
+ raise ValueError(f"visibility must be one of {', '.join(VISIBILITIES)}")
161
+ if import_method not in IMPORT_METHODS:
162
+ raise ValueError(f"import method must be one of {', '.join(IMPORT_METHODS)}")
163
+ bad = RESERVED_PROPERTIES.intersection(extra_properties or {})
164
+ if bad:
165
+ raise ValueError(f"properties reserved to oic-sync: {', '.join(sorted(bad))}")
166
+ self.glance = glance
167
+ self.http = http
168
+ self.compute = compute
169
+ self.volume = volume
170
+ self.visibility = visibility
171
+ self.import_method = import_method
172
+ self.extra_properties = dict(extra_properties or {})
173
+ self.optional_properties = optional_properties
174
+ self.import_timeout = import_timeout
175
+ self.poll_interval = poll_interval
176
+ self.sleep = sleep
177
+ self.clock = clock
178
+ self._description = description
179
+ self._methods: list[str] | None = None
180
+
181
+ def describe(self) -> str:
182
+ return self._description
183
+
184
+ # -- listing -------------------------------------------------------------------------
185
+
186
+ def list_managed(self, target: Target) -> list[ManagedImage]:
187
+ base = {"oic_id": target.image.id, "owner": self.glance.project_id}
188
+ raw: dict[str, dict[str, Any]] = {}
189
+ # Hidden images (os_hidden=true) are left out of the default list: query both.
190
+ for extra in ({}, {"os_hidden": "true"}):
191
+ for img in self.glance.list_images({**base, **extra}):
192
+ raw[img["id"]] = img
193
+ managed = []
194
+ for img in raw.values():
195
+ # Server-side property filters are exact, but check again: an old Glance or a
196
+ # proxy could ignore unknown filters.
197
+ if img.get("oic_id") != target.image.id:
198
+ continue
199
+ if img.get("owner") not in (None, self.glance.project_id):
200
+ continue
201
+ if (img.get("oic_variant") or None) != target.artifact.variant:
202
+ continue
203
+ managed.append(self._managed(img))
204
+ return managed
205
+
206
+ @staticmethod
207
+ def _managed(img: Mapping[str, Any]) -> ManagedImage:
208
+ return ManagedImage(
209
+ id=img["id"],
210
+ name=img.get("name") or "",
211
+ version=str(img.get("oic_version") or ""),
212
+ sha256=str(img.get("oic_sha256") or "").lower(),
213
+ status=img.get("status") or "unknown",
214
+ hidden=bool(img.get("os_hidden", False)),
215
+ created_at=_parse_time(img.get("created_at")),
216
+ visibility=img.get("visibility") or "private",
217
+ )
218
+
219
+ # -- import --------------------------------------------------------------------------
220
+
221
+ def properties(self, target: Target) -> dict[str, str]:
222
+ """Glance properties for the artifact: index set, variant, options, then overrides."""
223
+ artifact = target.artifact
224
+ props: dict[str, str] = {}
225
+ if self.optional_properties:
226
+ props.update(target.image.optional_glance_properties)
227
+ props.update(artifact.glance_properties)
228
+ props.update(self.extra_properties)
229
+ props["oic_id"] = target.image.id
230
+ props["oic_version"] = artifact.version
231
+ props["oic_sha256"] = artifact.sha256
232
+ if artifact.variant:
233
+ props["oic_variant"] = artifact.variant
234
+ return props
235
+
236
+ def create_body(self, target: Target, name: str) -> dict[str, Any]:
237
+ body: dict[str, Any] = {
238
+ "name": name,
239
+ "disk_format": target.artifact.format,
240
+ "container_format": "bare",
241
+ "visibility": self.visibility,
242
+ "os_hidden": True,
243
+ **self.properties(target),
244
+ }
245
+ if target.artifact.min_disk_gib:
246
+ body["min_disk"] = target.artifact.min_disk_gib
247
+ return body
248
+
249
+ def _available_methods(self) -> list[str]:
250
+ if self._methods is None:
251
+ self._methods = self.glance.import_methods()
252
+ log.info("Glance import methods: %s", ", ".join(self._methods) or "none")
253
+ return self._methods
254
+
255
+ def choose_method(self, target: Target) -> str:
256
+ methods = self._available_methods()
257
+ if self.import_method == AUTO:
258
+ # web-download is only usable when the result can be checked: Glance hashes with
259
+ # sha512 by default, which needs the index sha512.
260
+ if WEB_DOWNLOAD in methods and target.artifact.sha512:
261
+ return WEB_DOWNLOAD
262
+ return UPLOAD
263
+ if self.import_method in (WEB_DOWNLOAD, GLANCE_DIRECT) and self.import_method not in (
264
+ methods
265
+ ):
266
+ raise BackendError(f"this cloud does not offer the {self.import_method} import")
267
+ return self.import_method
268
+
269
+ def import_target(self, target: Target, name: str) -> ManagedImage:
270
+ method = self.choose_method(target)
271
+ try:
272
+ return self._import_with(target, name, method)
273
+ except HashMismatch:
274
+ raise
275
+ except BackendError as e:
276
+ if not (self.import_method == AUTO and method == WEB_DOWNLOAD):
277
+ raise
278
+ # A cloud may advertise web-download and still fail it at the store (seen on
279
+ # Infomaniak: "Glance import failed (stores: swift)"). In auto mode, fall back to
280
+ # a streaming upload, and keep to it for the rest of the run.
281
+ log.warning("%s: web-download failed (%s), falling back to upload", target.label, e)
282
+ self._methods = [m for m in self._available_methods() if m != WEB_DOWNLOAD]
283
+ return self._import_with(target, name, UPLOAD)
284
+
285
+ def _import_with(self, target: Target, name: str, method: str) -> ManagedImage:
286
+ log.info("%s: importing with %s", target.label, method)
287
+ created = self.glance.create_image(self.create_body(target, name))
288
+ image_id = created["id"]
289
+ try:
290
+ locally_verified = False
291
+ if method == WEB_DOWNLOAD:
292
+ self.glance.import_image(image_id, WEB_DOWNLOAD, target.artifact.url)
293
+ elif method == GLANCE_DIRECT:
294
+ self._stream(target, lambda data: self.glance.stage(image_id, data))
295
+ locally_verified = True
296
+ self.glance.import_image(image_id, GLANCE_DIRECT)
297
+ else:
298
+ self._stream(target, lambda data: self.glance.upload(image_id, data))
299
+ locally_verified = True
300
+ img = self._wait_active(image_id)
301
+ self._verify_glance(target, img, locally_verified)
302
+ return self._managed(img)
303
+ except BaseException:
304
+ log.warning("%s: removing incomplete image %s", target.label, image_id)
305
+ try:
306
+ self.glance.delete_image(image_id)
307
+ except BackendError as e:
308
+ log.error("%s: could not remove image %s: %s", target.label, image_id, e)
309
+ raise
310
+
311
+ def _stream(self, target: Target, send: Callable[[Any], None]) -> None:
312
+ """Download the artifact and hand it to Glance, hashing on the fly."""
313
+ artifact = target.artifact
314
+ try:
315
+ with self.http.stream("GET", artifact.url) as response:
316
+ response.raise_for_status()
317
+ length = response.headers.get("Content-Length")
318
+ if (
319
+ length is not None
320
+ and artifact.size is not None
321
+ and int(length) != (artifact.size)
322
+ ):
323
+ raise HashMismatch(
324
+ f"server announces {length} bytes, index says {artifact.size}"
325
+ )
326
+ chunks: Iterator[bytes] = response.iter_bytes(CHUNK)
327
+ reader = HashingReader(chunks, artifact.size)
328
+ send(reader)
329
+ # Drain anything Glance did not read (it should have read everything).
330
+ while reader.read(CHUNK):
331
+ pass
332
+ except httpx.HTTPError as e:
333
+ raise BackendError(f"download of {artifact.url} failed: {e}") from e
334
+ reader.verify(artifact.sha256, artifact.sha512, artifact.size)
335
+
336
+ def _wait_active(self, image_id: str) -> dict[str, Any]:
337
+ deadline = self.clock() + self.import_timeout
338
+ while True:
339
+ img = self.glance.get_image(image_id)
340
+ status = img.get("status")
341
+ failed_stores = img.get("os_glance_failed_import")
342
+ if failed_stores:
343
+ raise BackendError(f"Glance import failed (stores: {failed_stores})")
344
+ if status == "active":
345
+ return img
346
+ if status in ("killed", "deleted", "pending_delete"):
347
+ raise BackendError(f"image {image_id} ended in status {status}")
348
+ if self.clock() >= deadline:
349
+ raise BackendError(
350
+ f"image {image_id} not active after {self.import_timeout:.0f}s ({status})"
351
+ )
352
+ self.sleep(self.poll_interval)
353
+
354
+ @staticmethod
355
+ def _verify_glance(target: Target, img: Mapping[str, Any], locally_verified: bool) -> None:
356
+ """Compare what Glance stored with the index (os_hash_algo/os_hash_value, size)."""
357
+ artifact = target.artifact
358
+ algo = (img.get("os_hash_algo") or "").lower()
359
+ value = (img.get("os_hash_value") or "").lower()
360
+ expected = {"sha512": artifact.sha512, "sha256": artifact.sha256}.get(algo)
361
+ if expected and value:
362
+ if value != expected:
363
+ raise HashMismatch(f"Glance computed {algo} {value}, index says {expected}")
364
+ elif not locally_verified:
365
+ raise HashMismatch(
366
+ f"cannot verify the imported data: Glance reports {algo or 'no'} multihash"
367
+ " and the index has no matching digest"
368
+ )
369
+ size = img.get("size")
370
+ if artifact.size is not None and size is not None and int(size) != artifact.size:
371
+ raise HashMismatch(f"Glance stored {size} bytes, index says {artifact.size}")
372
+
373
+ # -- rotation ------------------------------------------------------------------------
374
+
375
+ def rename(self, image: ManagedImage, name: str, hidden: bool) -> None:
376
+ self.glance.update_image(
377
+ image.id,
378
+ [
379
+ {"op": "replace", "path": "/name", "value": name},
380
+ {"op": "replace", "path": "/os_hidden", "value": hidden},
381
+ ],
382
+ )
383
+
384
+ def delete(self, image: ManagedImage) -> None:
385
+ self.glance.delete_image(image.id)
386
+
387
+ def in_use(self, image: ManagedImage) -> bool | None:
388
+ """True when a server or a volume references the image; None when unknown.
389
+
390
+ Images other projects can see (public, community, shared) need an admin view of
391
+ every project (all_tenants); a refusal means "unknown", so nothing is deleted.
392
+ """
393
+ all_projects = image.visibility != "private"
394
+ scope: dict[str, Any] = {"all_tenants": 1} if all_projects else {}
395
+ if self.compute is None:
396
+ return None
397
+ try:
398
+ response = self.compute.get("/servers", params={"image": image.id, "limit": 1, **scope})
399
+ if response.status_code != 200:
400
+ log.warning("server list for %s: HTTP %s", image.id, response.status_code)
401
+ return None
402
+ if response.json().get("servers"):
403
+ return True
404
+ if self.volume is not None:
405
+ response = self.volume.get(
406
+ "/volumes",
407
+ params={
408
+ "glance_metadata": json.dumps({"image_id": image.id}),
409
+ "limit": 1,
410
+ **scope,
411
+ },
412
+ microversion="3.4",
413
+ )
414
+ if response.status_code != 200:
415
+ log.warning("volume list for %s: HTTP %s", image.id, response.status_code)
416
+ return None
417
+ if response.json().get("volumes"):
418
+ return True
419
+ except Exception as e:
420
+ log.warning("usage check for %s failed: %s", image.id, e)
421
+ return None
422
+ return False
423
+
424
+
425
+ def connect(
426
+ cloud: str | None,
427
+ region: str | None,
428
+ http: httpx.Client,
429
+ **backend_options: Any,
430
+ ) -> OpenStackBackend:
431
+ """Build the backend from clouds.yaml (or OS_* variables when no cloud is named)."""
432
+ try:
433
+ import openstack
434
+ except ImportError as e:
435
+ raise BackendError(
436
+ "the OpenStack backend needs openstacksdk: pip install 'oic-sync[openstack]'"
437
+ ) from e
438
+
439
+ from oic_sync import __version__
440
+
441
+ conn = openstack.connect(
442
+ cloud=cloud, region_name=region, app_name="oic-sync", app_version=__version__
443
+ )
444
+ project_id = conn.current_project_id
445
+ if not project_id:
446
+ raise BackendError("cannot determine the current project (scoped token required)")
447
+ volume = conn.block_storage if conn.has_service("block-storage") else None
448
+ compute = conn.compute if conn.has_service("compute") else None
449
+ where = f"cloud {cloud or '(environment)'}"
450
+ region_name = region or getattr(conn.config, "region_name", None)
451
+ if region_name:
452
+ where += f", region {region_name}"
453
+ where += f", project {project_id}"
454
+ return OpenStackBackend(
455
+ Glance(conn.image, project_id),
456
+ http,
457
+ compute=compute,
458
+ volume=volume,
459
+ description=where,
460
+ **backend_options,
461
+ )