oic-sync 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oic_sync/engine.py ADDED
@@ -0,0 +1,331 @@
1
+ """Stateless, idempotent reconciliation of index targets with a platform.
2
+
3
+ For each target (an artifact of the index), the engine lists the images the backend recognises
4
+ as ours, plans what to change, then executes it:
5
+
6
+ 1. import the artifact when no usable image has its sha256 (after a cosign check);
7
+ 2. give the stable name ("Debian 13") to that image and unhide it;
8
+ 3. retire older versions: rename them "<stable name> (<version>)" and hide them (os_hidden),
9
+ so they stay bootable by ID but leave image lists;
10
+ 4. keep the `keep` newest versions (the current one included); older ones are deleted only
11
+ with `delete`, and only when nothing uses them;
12
+ 5. remove failed or abandoned imports.
13
+
14
+ Nothing is stored locally: running it twice in a row changes nothing the second time.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import logging
20
+ from collections.abc import Callable, Iterable, Sequence
21
+ from dataclasses import dataclass, field
22
+ from datetime import datetime, timedelta, timezone
23
+ from typing import Protocol
24
+
25
+ from oic_sync.backends.base import FAILED, TRANSIENT, Backend, BackendError, ManagedImage
26
+ from oic_sync.hashing import HashMismatch
27
+ from oic_sync.index import Target
28
+ from oic_sync.signature import SignatureError, SignatureVerifier
29
+
30
+ log = logging.getLogger(__name__)
31
+
32
+ # An import that has not finished after this long is considered abandoned (crashed run).
33
+ DEFAULT_STALE_AFTER = timedelta(hours=6)
34
+
35
+ IMPORT = "import"
36
+ PROMOTE = "promote"
37
+ RETIRE = "retire"
38
+ DELETE = "delete"
39
+ CLEANUP = "cleanup"
40
+
41
+ UP_TO_DATE = "up-to-date"
42
+ NEEDS_ROTATION = "needs-rotation"
43
+ OUTDATED = "outdated"
44
+ MISSING = "missing"
45
+ IMPORTING = "importing"
46
+
47
+
48
+ def versioned_name(stable: str, version: str, sha256: str | None = None) -> str:
49
+ """Name of a retired version; the digest prefix disambiguates a republished version."""
50
+ if sha256:
51
+ return f"{stable} ({version}, {sha256[:12]})"
52
+ return f"{stable} ({version})"
53
+
54
+
55
+ class Naming(Protocol):
56
+ """How a platform names the image holding the stable name and the retired versions.
57
+
58
+ A backend may expose its own as a `naming` attribute (Proxmox VE needs DNS names);
59
+ otherwise the engine uses GlanceNaming.
60
+ """
61
+
62
+ def stable(self, target: Target) -> str: ...
63
+
64
+ def versioned(self, target: Target, version: str, sha256: str | None = None) -> str: ...
65
+
66
+
67
+ class GlanceNaming:
68
+ """The current version is "Debian 13", older ones "Debian 13 (13-20261026-2643)"."""
69
+
70
+ def stable(self, target: Target) -> str:
71
+ return target.stable_name
72
+
73
+ def versioned(self, target: Target, version: str, sha256: str | None = None) -> str:
74
+ return versioned_name(target.stable_name, version, sha256)
75
+
76
+
77
+ DEFAULT_NAMING: Naming = GlanceNaming()
78
+
79
+
80
+ def naming_of(backend: object) -> Naming:
81
+ """The naming scheme of a backend: its `naming` attribute, else the Glance one."""
82
+ naming = getattr(backend, "naming", None)
83
+ return naming if naming is not None else DEFAULT_NAMING
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class Action:
88
+ kind: str
89
+ image: ManagedImage | None = None
90
+ name: str | None = None
91
+
92
+ def describe(self) -> str:
93
+ if self.kind == IMPORT:
94
+ return f"import as '{self.name}' (hidden until verified)"
95
+ assert self.image is not None
96
+ ref = f"{self.image.id} '{self.image.name}'"
97
+ if self.kind == PROMOTE:
98
+ return f"promote {ref} to '{self.name}'"
99
+ if self.kind == RETIRE:
100
+ return f"retire {ref} as '{self.name}' (hidden)"
101
+ if self.kind == DELETE:
102
+ return f"delete {ref} (beyond --keep, if unused)"
103
+ return f"remove failed import {ref} (status {self.image.status})"
104
+
105
+
106
+ @dataclass
107
+ class Plan:
108
+ target: Target
109
+ state: str
110
+ current: ManagedImage | None
111
+ older: list[ManagedImage]
112
+ actions: list[Action]
113
+ notes: list[str] = field(default_factory=list)
114
+ stable_name: str = ""
115
+
116
+ def has(self, kind: str) -> bool:
117
+ return any(a.kind == kind for a in self.actions)
118
+
119
+
120
+ def _retire_name(
121
+ target: Target, image: ManagedImage, others: Iterable[ManagedImage], naming: Naming
122
+ ) -> str:
123
+ clash = image.version == target.artifact.version or any(
124
+ o.id != image.id and o.version == image.version and o.sha256 != image.sha256 for o in others
125
+ )
126
+ return naming.versioned(target, image.version, image.sha256 if clash else None)
127
+
128
+
129
+ def plan_target(
130
+ target: Target,
131
+ images: Sequence[ManagedImage],
132
+ *,
133
+ keep: int,
134
+ delete: bool,
135
+ now: datetime,
136
+ stale_after: timedelta = DEFAULT_STALE_AFTER,
137
+ naming: Naming = DEFAULT_NAMING,
138
+ ) -> Plan:
139
+ """Decide what to do for one target, from the images the platform holds for it."""
140
+ if keep < 1:
141
+ raise ValueError("keep must be at least 1")
142
+ sha = target.artifact.sha256
143
+ stable = naming.stable(target)
144
+ actions: list[Action] = []
145
+ notes: list[str] = []
146
+
147
+ def stale(m: ManagedImage) -> bool:
148
+ return m.status in FAILED or (m.status in TRANSIENT and now - m.created_at > stale_after)
149
+
150
+ newest_first = sorted(images, key=lambda m: m.created_at, reverse=True)
151
+ failed = [m for m in newest_first if stale(m)]
152
+ in_progress = [m for m in newest_first if m.status in TRANSIENT and not stale(m)]
153
+ current = next((m for m in newest_first if m.usable and m.sha256 == sha), None)
154
+ older = [m for m in newest_first if m.usable and m is not current]
155
+
156
+ actions.extend(Action(CLEANUP, m) for m in failed)
157
+
158
+ importing_elsewhere = current is None and any(m.sha256 == sha for m in in_progress)
159
+ if current is None and not importing_elsewhere:
160
+ actions.append(Action(IMPORT, name=naming.versioned(target, target.artifact.version)))
161
+ if current is not None and (current.name != stable or current.hidden):
162
+ actions.append(Action(PROMOTE, current, stable))
163
+
164
+ if importing_elsewhere:
165
+ notes.append("an import of this version is already in progress; nothing else done")
166
+ else:
167
+ for rank, m in enumerate(older, start=1):
168
+ name = _retire_name(target, m, older, naming)
169
+ if m.name != name or not m.hidden:
170
+ actions.append(Action(RETIRE, m, name))
171
+ if rank >= keep: # the current version counts as the first kept one
172
+ if delete:
173
+ actions.append(Action(DELETE, m))
174
+ else:
175
+ notes.append(f"{m.id} ({m.version}) is beyond --keep {keep}; use --delete")
176
+
177
+ if current is not None:
178
+ state = UP_TO_DATE if not actions else NEEDS_ROTATION
179
+ elif importing_elsewhere:
180
+ state = IMPORTING
181
+ elif older:
182
+ state = OUTDATED
183
+ else:
184
+ state = MISSING
185
+ return Plan(target, state, current, older, actions, notes, stable_name=stable)
186
+
187
+
188
+ @dataclass
189
+ class TargetResult:
190
+ target: Target
191
+ state: str
192
+ ok: bool = True
193
+ messages: list[str] = field(default_factory=list)
194
+
195
+
196
+ class Engine:
197
+ def __init__(
198
+ self,
199
+ backend: Backend,
200
+ *,
201
+ verifier: SignatureVerifier | None,
202
+ keep: int = 2,
203
+ delete: bool = False,
204
+ dry_run: bool = False,
205
+ out: Callable[[str], None] = print,
206
+ now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
207
+ stale_after: timedelta = DEFAULT_STALE_AFTER,
208
+ ) -> None:
209
+ if keep < 1:
210
+ raise ValueError("keep must be at least 1")
211
+ self.backend = backend
212
+ self.verifier = verifier
213
+ self.keep = keep
214
+ self.delete = delete
215
+ self.dry_run = dry_run
216
+ self.out = out
217
+ self.now = now
218
+ self.stale_after = stale_after
219
+ self.naming = naming_of(backend)
220
+
221
+ def plan(self, target: Target) -> Plan:
222
+ images = self.backend.list_managed(target)
223
+ return plan_target(
224
+ target,
225
+ images,
226
+ keep=self.keep,
227
+ delete=self.delete,
228
+ now=self.now(),
229
+ stale_after=self.stale_after,
230
+ naming=self.naming,
231
+ )
232
+
233
+ def status(self, targets: Iterable[Target]) -> list[Plan]:
234
+ return [self.plan(t) for t in targets]
235
+
236
+ def sync(self, targets: Iterable[Target]) -> list[TargetResult]:
237
+ results = []
238
+ for target in targets:
239
+ result = self._sync_one(target)
240
+ results.append(result)
241
+ return results
242
+
243
+ def _say(self, result: TargetResult, message: str) -> None:
244
+ result.messages.append(message)
245
+ self.out(f"{result.target.label}: {message}")
246
+
247
+ def _sync_one(self, target: Target) -> TargetResult:
248
+ try:
249
+ plan = self.plan(target)
250
+ except Exception as e:
251
+ result = TargetResult(target, "error", ok=False)
252
+ self._say(result, f"ERROR listing images: {e}")
253
+ return result
254
+ result = TargetResult(target, plan.state)
255
+ prefix = "would " if self.dry_run else ""
256
+ if not plan.actions and plan.state == UP_TO_DATE:
257
+ self._say(result, f"up to date ({target.artifact.version})")
258
+ for note in plan.notes:
259
+ self._say(result, f"note: {note}")
260
+ if not plan.actions:
261
+ return result
262
+
263
+ if plan.has(IMPORT):
264
+ if self.verifier is None:
265
+ self._say(result, "WARNING: cosign signature NOT verified (--no-verify-signature)")
266
+ else:
267
+ try:
268
+ identity = self.verifier.verify(target.artifact)
269
+ except SignatureError as e:
270
+ result.ok = False
271
+ self._say(result, f"ERROR: {e}; refusing to import {target.artifact.version}")
272
+ return result
273
+ self._say(result, f"signature verified ({identity})")
274
+
275
+ try:
276
+ self._execute(plan, result, prefix)
277
+ except (BackendError, HashMismatch) as e:
278
+ result.ok = False
279
+ self._say(result, f"ERROR: {e}")
280
+ except Exception as e: # keep going with the other targets
281
+ log.debug("unexpected error", exc_info=True)
282
+ result.ok = False
283
+ self._say(result, f"ERROR: {type(e).__name__}: {e}")
284
+ return result
285
+
286
+ def _execute(self, plan: Plan, result: TargetResult, prefix: str) -> None:
287
+ stable = plan.stable_name or self.naming.stable(plan.target)
288
+ by_kind: dict[str, list[Action]] = {}
289
+ for a in plan.actions:
290
+ by_kind.setdefault(a.kind, []).append(a)
291
+
292
+ for a in by_kind.get(CLEANUP, []):
293
+ self._say(result, prefix + a.describe())
294
+ if not self.dry_run and a.image is not None:
295
+ self.backend.delete(a.image)
296
+
297
+ for a in by_kind.get(IMPORT, []):
298
+ self._say(result, prefix + a.describe() + f": {plan.target.artifact.version}")
299
+ if not self.dry_run:
300
+ assert a.name is not None
301
+ new = self.backend.import_target(plan.target, a.name)
302
+ self._say(result, f"imported {new.id}, digests verified")
303
+ self.backend.rename(new, stable, hidden=False)
304
+ self._say(result, f"promoted {new.id} to '{stable}'")
305
+ else:
306
+ self._say(result, f"would promote it to '{stable}'")
307
+
308
+ for a in by_kind.get(PROMOTE, []):
309
+ self._say(result, prefix + a.describe())
310
+ if not self.dry_run and a.image is not None and a.name is not None:
311
+ self.backend.rename(a.image, a.name, hidden=False)
312
+
313
+ for a in by_kind.get(RETIRE, []):
314
+ self._say(result, prefix + a.describe())
315
+ if not self.dry_run and a.image is not None and a.name is not None:
316
+ self.backend.rename(a.image, a.name, hidden=True)
317
+
318
+ for a in by_kind.get(DELETE, []):
319
+ assert a.image is not None
320
+ used = self.backend.in_use(a.image)
321
+ if used is None:
322
+ self._say(result, f"keep {a.image.id}: cannot tell whether it is in use")
323
+ continue
324
+ if used:
325
+ self._say(
326
+ result, f"keep {a.image.id}: used by an instance, a volume or a linked clone"
327
+ )
328
+ continue
329
+ self._say(result, prefix + a.describe())
330
+ if not self.dry_run:
331
+ self.backend.delete(a.image)
oic_sync/hashing.py ADDED
@@ -0,0 +1,82 @@
1
+ """Streaming reader that hashes what it passes through."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from collections.abc import Iterator
7
+
8
+
9
+ class HashMismatch(Exception):
10
+ """The downloaded data does not match the digests or size of the index."""
11
+
12
+
13
+ class HashingReader:
14
+ """File-like wrapper around a byte-chunk iterator, computing sha256 and sha512 on the fly.
15
+
16
+ It is handed to requests (via keystoneauth) as an upload body: requests reads it in blocks
17
+ and, when the expected size is known, sends a Content-Length instead of a chunked body
18
+ (exposed through the `len` attribute, which requests' super_len looks for).
19
+ Reading more than the expected size aborts the upload at once.
20
+ """
21
+
22
+ def __init__(self, chunks: Iterator[bytes], expected_size: int | None = None) -> None:
23
+ self._chunks = chunks
24
+ self._buffer = b""
25
+ self._done = False
26
+ self.expected_size = expected_size
27
+ self.bytes_read = 0
28
+ self._sha256 = hashlib.sha256()
29
+ self._sha512 = hashlib.sha512()
30
+ if expected_size is not None:
31
+ self.len = expected_size
32
+
33
+ def _account(self, data: bytes) -> bytes:
34
+ if data:
35
+ self.bytes_read += len(data)
36
+ if self.expected_size is not None and self.bytes_read > self.expected_size:
37
+ raise HashMismatch(
38
+ f"download is larger than the {self.expected_size} bytes of the index"
39
+ )
40
+ self._sha256.update(data)
41
+ self._sha512.update(data)
42
+ return data
43
+
44
+ def read(self, size: int = -1) -> bytes:
45
+ if size is None or size < 0:
46
+ parts = [self._buffer]
47
+ self._buffer = b""
48
+ if not self._done:
49
+ parts.extend(self._chunks)
50
+ self._done = True
51
+ return self._account(b"".join(parts))
52
+ while len(self._buffer) < size and not self._done:
53
+ try:
54
+ self._buffer += next(self._chunks)
55
+ except StopIteration:
56
+ self._done = True
57
+ data, self._buffer = self._buffer[:size], self._buffer[size:]
58
+ return self._account(data)
59
+
60
+ def __iter__(self) -> Iterator[bytes]:
61
+ while True:
62
+ block = self.read(1024 * 1024)
63
+ if not block:
64
+ return
65
+ yield block
66
+
67
+ @property
68
+ def sha256(self) -> str:
69
+ return self._sha256.hexdigest()
70
+
71
+ @property
72
+ def sha512(self) -> str:
73
+ return self._sha512.hexdigest()
74
+
75
+ def verify(self, sha256: str, sha512: str | None, size: int | None) -> None:
76
+ """Raise HashMismatch unless everything was read and matches the index."""
77
+ if size is not None and self.bytes_read != size:
78
+ raise HashMismatch(f"size mismatch: read {self.bytes_read} bytes, index says {size}")
79
+ if self.sha256 != sha256:
80
+ raise HashMismatch(f"sha256 mismatch: computed {self.sha256}, index says {sha256}")
81
+ if sha512 is not None and self.sha512 != sha512:
82
+ raise HashMismatch(f"sha512 mismatch: computed {self.sha512}, index says {sha512}")
oic_sync/http.py ADDED
@@ -0,0 +1,27 @@
1
+ """HTTP client shared by the index, bundle and image downloads."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import httpx
6
+
7
+ from oic_sync import USER_AGENT
8
+
9
+ # Large reads (qcow2 streaming into Glance) can stall while Glance writes to its store:
10
+ # keep a generous read timeout, but fail fast on connect.
11
+ DEFAULT_TIMEOUT = httpx.Timeout(connect=15.0, read=300.0, write=60.0, pool=30.0)
12
+
13
+
14
+ def make_client(
15
+ *, transport: httpx.BaseTransport | None = None, timeout: httpx.Timeout | None = None
16
+ ) -> httpx.Client:
17
+ """httpx client with the generic oic-sync User-Agent (no personal data) and retries.
18
+
19
+ Cloudflare in front of images.openimages.cloud rejects some default library agents,
20
+ so the User-Agent is always set explicitly.
21
+ """
22
+ return httpx.Client(
23
+ headers={"User-Agent": USER_AGENT},
24
+ timeout=timeout or DEFAULT_TIMEOUT,
25
+ follow_redirects=True,
26
+ transport=transport or httpx.HTTPTransport(retries=3),
27
+ )