pixie-lab 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. pixie/__init__.py +20 -0
  2. pixie/_util.py +38 -0
  3. pixie/catalog/__init__.py +25 -0
  4. pixie/catalog/_fetcher.py +513 -0
  5. pixie/catalog/_routes.py +392 -0
  6. pixie/catalog/_schema.py +263 -0
  7. pixie/catalog/_store.py +234 -0
  8. pixie/deploy/__init__.py +26 -0
  9. pixie/deploy/_main.py +321 -0
  10. pixie/deploy/_templates.py +153 -0
  11. pixie/disks.py +85 -0
  12. pixie/events/__init__.py +28 -0
  13. pixie/events/_kinds.py +203 -0
  14. pixie/events/_log.py +210 -0
  15. pixie/events/_routes.py +42 -0
  16. pixie/exports/__init__.py +17 -0
  17. pixie/exports/_routes.py +211 -0
  18. pixie/exports/_store.py +156 -0
  19. pixie/exports/_supervisor.py +240 -0
  20. pixie/flash.py +1971 -0
  21. pixie/images.py +473 -0
  22. pixie/machines/__init__.py +16 -0
  23. pixie/machines/_routes.py +190 -0
  24. pixie/machines/_store.py +623 -0
  25. pixie/oras.py +547 -0
  26. pixie/pivot/__init__.py +180 -0
  27. pixie/pivot/nbdboot +348 -0
  28. pixie/pxe/__init__.py +19 -0
  29. pixie/pxe/_renderer.py +244 -0
  30. pixie/pxe/_routes.py +386 -0
  31. pixie/tftp/__init__.py +21 -0
  32. pixie/tftp/_supervisor.py +129 -0
  33. pixie/tui/__init__.py +185 -0
  34. pixie/tui/_app.py +2219 -0
  35. pixie/tui_catalog.py +657 -0
  36. pixie/web/__init__.py +6 -0
  37. pixie/web/_auth.py +70 -0
  38. pixie/web/_settings_store.py +211 -0
  39. pixie/web/_static/.gitkeep +0 -0
  40. pixie/web/_static/bootstrap-icons.min.css +5 -0
  41. pixie/web/_static/bootstrap.min.css +12 -0
  42. pixie/web/_static/fonts/bootstrap-icons.woff +0 -0
  43. pixie/web/_static/fonts/bootstrap-icons.woff2 +0 -0
  44. pixie/web/_static/htmx.min.js +1 -0
  45. pixie/web/_static/pixie-favicon.png +0 -0
  46. pixie/web/_static/sse.js +290 -0
  47. pixie/web/_table_state.py +261 -0
  48. pixie/web/_templates/_partials/page_description.html +22 -0
  49. pixie/web/_templates/_partials/recent_events.html +47 -0
  50. pixie/web/_templates/_partials/table_helpers.html +189 -0
  51. pixie/web/_templates/catalog.html +364 -0
  52. pixie/web/_templates/catalog_detail.html +386 -0
  53. pixie/web/_templates/dashboard.html +256 -0
  54. pixie/web/_templates/events.html +125 -0
  55. pixie/web/_templates/ipxe/bootstrap.j2 +12 -0
  56. pixie/web/_templates/ipxe/exit.j2 +10 -0
  57. pixie/web/_templates/ipxe/nbdboot.j2 +57 -0
  58. pixie/web/_templates/ipxe/pixie-live-env.j2 +55 -0
  59. pixie/web/_templates/ipxe/unavailable.j2 +14 -0
  60. pixie/web/_templates/layout.html +345 -0
  61. pixie/web/_templates/login.html +40 -0
  62. pixie/web/_templates/machine_detail.html +786 -0
  63. pixie/web/_templates/machines.html +186 -0
  64. pixie/web/_templates/settings.html +265 -0
  65. pixie/web/main.py +1910 -0
  66. pixie_lab-0.1.0.dist-info/METADATA +107 -0
  67. pixie_lab-0.1.0.dist-info/RECORD +70 -0
  68. pixie_lab-0.1.0.dist-info/WHEEL +4 -0
  69. pixie_lab-0.1.0.dist-info/entry_points.txt +3 -0
  70. pixie_lab-0.1.0.dist-info/licenses/LICENSE +674 -0
pixie/flash.py ADDED
@@ -0,0 +1,1971 @@
1
+ """Flash plan: validate that an image can be written to a target disk.
2
+
3
+ Split into three layers so unit tests don't need to mock anything to
4
+ cover the validation logic:
5
+
6
+ - ``probe_image`` and ``probe_target`` do the I/O (reading file stats,
7
+ shelling out to ``qemu-img info``, ``zstd -l``, ``lsblk``) and return
8
+ plain :class:`ImageInfo` / :class:`TargetInfo` dataclasses.
9
+ - ``make_plan`` is pure: it bundles probed info into a :class:`FlashPlan`.
10
+ - ``validate_plan`` is pure: it returns a list of error strings.
11
+ - ``execute_plan`` does the destructive write (qemu-img convert /
12
+ zstd -d / dd as appropriate for the image format). pixie has no
13
+ post-flash provisioning step -- first-boot bring-up belongs in
14
+ the image builder (cloud-init / NoCloud); pixie only writes bytes.
15
+
16
+ The ``pixie`` wizard calls all four. Tests construct ``ImageInfo`` /
17
+ ``TargetInfo`` directly and exercise ``make_plan`` / ``validate_plan``
18
+ without mocks. The probe and write functions have their own targeted
19
+ tests for the subprocess-shelling-out parts; integration tests against
20
+ a real loop device live in ``tests/test_flash_integration.py``.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import contextlib
26
+ import json
27
+ import re
28
+ import shutil
29
+ import stat
30
+ import subprocess
31
+ import threading
32
+ import time
33
+ from collections.abc import Callable
34
+ from dataclasses import dataclass, field
35
+ from pathlib import Path
36
+ from typing import IO, Any, TypeAlias
37
+
38
+ from pixie import images, oras
39
+
40
+ # ``cancel`` callbacks return True to abort an in-flight flash. The
41
+ # flash code polls ~4Hz from a watchdog thread; on True it terminates
42
+ # all child subprocesses (curl + decompressor + dd) and the main
43
+ # pipeline raises :class:`FlashCancelled`.
44
+ CancelCheck: TypeAlias = Callable[[], bool]
45
+
46
+
47
+ @dataclass
48
+ class FlashProgress:
49
+ """One lifecycle event from :func:`execute_plan` / ``cmd_flash``.
50
+
51
+ The ``event`` field is a stable string callers dispatch on. Current
52
+ events:
53
+
54
+ - ``started`` - flash beginning; ``total_bytes`` is the
55
+ image's virtual size when known.
56
+ - ``writing`` - about to invoke the format-specific
57
+ writer (``dd`` / ``zstd | dd`` / ``qemu-img convert``).
58
+ - ``writing_progress`` - byte-level progress from the running
59
+ writer; ``bytes_written`` is set, ``total_bytes`` carries
60
+ through from ``started`` so consumers can compute percent /
61
+ ETA without holding state. Emitted ~1/sec from a daemon
62
+ thread that parses ``dd``'s ``status=progress`` stderr.
63
+ - ``downloading_progress`` - byte-level progress from the
64
+ network-side ``dd`` spliced between ``curl`` and the rest
65
+ of the URL-streaming pipeline. ``bytes_downloaded`` is set
66
+ and ``total_bytes`` carries the **compressed** payload size
67
+ (curl's Content-Length / oras layer size); compare against
68
+ ``writing_progress``'s decompressed total to see network
69
+ throughput separately from disk-write throughput. Only
70
+ emitted on the URL-streaming flash paths
71
+ (``_flash_*_from_url``); local-file flashes have no
72
+ download stage.
73
+ - ``synced`` - kernel buffers flushed.
74
+ - ``partprobed`` - partition table re-read; flash
75
+ hardware-complete.
76
+ - ``done`` - emitted by ``cmd_flash`` after the
77
+ flash succeeded.
78
+ - ``failed`` - emitted on any :class:`FlashError`;
79
+ ``note`` carries the exception string. The exception is then
80
+ re-raised.
81
+ - ``subprocess_log`` - one line of stderr from an auxiliary
82
+ pipeline subprocess (``zstd`` / ``gzip`` / ``xz`` / ``bzip2`` /
83
+ ``curl``). ``note`` is the line, already prefixed with the
84
+ source label (e.g. ``"zstd: ..."``). The ``pixie`` wizard renders
85
+ these above its progress widget; callers without a progress
86
+ callback can ignore them (the subprocess's stderr is already
87
+ inherited in that mode). Live updates that use carriage-return-
88
+ only refresh (curl/zstd's own progress bars) don't show up
89
+ live -- the pump reads newline-terminated lines, so only the
90
+ final newline-terminated message lands here. That keeps the
91
+ Rich progress bar uncluttered while still surfacing real
92
+ errors + end-of-run stats.
93
+
94
+ ``total_bytes`` is set on ``started`` (the image's virtual /
95
+ decompressed size when known) and on ``writing_progress``.
96
+ On ``downloading_progress`` it carries the COMPRESSED payload
97
+ size, which differs from the decompressed total for compressed
98
+ images. ``bytes_written`` is set only on ``writing_progress``;
99
+ ``bytes_downloaded`` only on ``downloading_progress``.
100
+ """
101
+
102
+ event: str
103
+ note: str = ""
104
+ total_bytes: int | None = None
105
+ bytes_written: int | None = None
106
+ bytes_downloaded: int | None = None
107
+
108
+
109
+ ProgressCallback: TypeAlias = Callable[[FlashProgress], None]
110
+
111
+
112
+ def _emit(progress: ProgressCallback | None, event: str, **fields: Any) -> None:
113
+ """Call ``progress`` with a :class:`FlashProgress` if one was provided."""
114
+ if progress is None:
115
+ return
116
+ progress(FlashProgress(event=event, **fields))
117
+
118
+
119
+ # ``dd status=progress`` writes a periodic line to stderr like:
120
+ # 13312000 bytes (13 MB, 13 MiB) copied, 0.103 s, 129 MB/s
121
+ # preceded by a ``\r`` so terminals overwrite the prior line. We parse
122
+ # the leading byte count and emit a ``writing_progress`` event ~1/sec.
123
+ _DD_PROGRESS_RE = re.compile(r"^(\d+)\s+bytes\b")
124
+
125
+
126
+ def _pump_dd_progress(
127
+ stream: IO[str],
128
+ progress: ProgressCallback,
129
+ total_bytes: int | None,
130
+ *,
131
+ event: str = "writing_progress",
132
+ bytes_field: str = "bytes_written",
133
+ ) -> None:
134
+ """Read ``dd``'s stderr and emit byte-level progress events.
135
+
136
+ Designed to run in a daemon thread alongside a ``dd`` process.
137
+ Defaults emit the disk-write ``writing_progress`` events; pass
138
+ ``event="downloading_progress"`` + ``bytes_field="bytes_downloaded"``
139
+ when the dd being pumped is the network-side splice that counts
140
+ bytes coming out of curl.
141
+
142
+ ``dd`` separates progress lines with ``\\r`` (so each line
143
+ overwrites the previous one in a terminal); we replace ``\\r``
144
+ with ``\\n`` before splitting so we get one progress line per
145
+ chunk regardless of terminal-style behaviour.
146
+
147
+ Returns when ``stream`` closes (i.e. dd has exited).
148
+ """
149
+
150
+ def _publish(byte_count: int) -> None:
151
+ _emit(
152
+ progress,
153
+ event,
154
+ **{bytes_field: byte_count},
155
+ total_bytes=total_bytes,
156
+ )
157
+
158
+ buf = ""
159
+ while True:
160
+ chunk = stream.read(256)
161
+ if not chunk:
162
+ # Drain whatever's left in the buffer.
163
+ for line in buf.replace("\r", "\n").splitlines():
164
+ m = _DD_PROGRESS_RE.match(line.strip())
165
+ if m:
166
+ _publish(int(m.group(1)))
167
+ return
168
+ buf += chunk
169
+ # Use the LAST progress line in the buffer as the most recent
170
+ # snapshot. dd emits monotonically-increasing byte counts so
171
+ # rendering only the latest is fine.
172
+ lines = buf.replace("\r", "\n").splitlines()
173
+ if not lines:
174
+ continue
175
+ # Keep the partial trailing line for the next read.
176
+ if buf.endswith(("\n", "\r")):
177
+ buf = ""
178
+ else:
179
+ buf = lines[-1]
180
+ lines = lines[:-1]
181
+ # Find the most recent line that matches the byte-count pattern.
182
+ for line in reversed(lines):
183
+ m = _DD_PROGRESS_RE.match(line.strip())
184
+ if m:
185
+ _publish(int(m.group(1)))
186
+ break
187
+
188
+
189
+ _ZSTD_SIZE_UNITS: dict[str, int] = {
190
+ "B": 1,
191
+ "KiB": 1024,
192
+ "MiB": 1024**2,
193
+ "GiB": 1024**3,
194
+ "TiB": 1024**4,
195
+ }
196
+
197
+ _ZSTD_SIZE_RE = re.compile(r"([\d.]+)\s+(B|KiB|MiB|GiB|TiB)")
198
+
199
+
200
+ @dataclass
201
+ class ImageInfo:
202
+ """Probed metadata for an image source.
203
+
204
+ Either ``path`` (a local file) or ``url`` (an HTTP/HTTPS or
205
+ ``oras://`` reference) is set; never both. URL-sourced images
206
+ stream through curl directly to the target disk for ``.img`` /
207
+ ``.img.{gz,zst,xz,bz2}`` (no temp file); for ``.qcow2`` they get
208
+ downloaded to a temp file first because qcow2 is random-access.
209
+ ``oras://`` URLs go through :mod:`withcache.oras` first to resolve the
210
+ layer digest and inject a bearer-token Authorization header into
211
+ the curl call.
212
+ """
213
+
214
+ path: Path | None
215
+ format: str | None
216
+ size_bytes: int
217
+ virtual_size_bytes: int | None # what would be written to disk; None = unknown
218
+ url: str | None = None
219
+ # Declared content digest (``sha256:<hex>``) for a URL source whose
220
+ # catalog / server committed to one. Verified on the wire during the
221
+ # flash. ``oras://`` refs resolve their own digest at flash time, so
222
+ # this stays ``None`` for them; it carries the catalog ``sha256`` for
223
+ # plain HTTP sources. ``None`` => no declared digest to check.
224
+ expected_sha: str | None = None
225
+
226
+ @property
227
+ def display(self) -> str:
228
+ """User-facing identifier (URL or path string)."""
229
+ return self.url if self.url is not None else str(self.path)
230
+
231
+
232
+ @dataclass
233
+ class TargetInfo:
234
+ """Probed metadata for a candidate target."""
235
+
236
+ path: Path
237
+ exists: bool
238
+ is_block_device: bool
239
+ size_bytes: int | None
240
+ mountpoints: list[str]
241
+
242
+
243
+ @dataclass
244
+ class FlashPlan:
245
+ """Inputs and computed metadata for a flash operation."""
246
+
247
+ image: ImageInfo
248
+ target: TargetInfo
249
+ notes: list[str] = field(default_factory=list)
250
+
251
+ def to_dict(self) -> dict[str, Any]:
252
+ return {
253
+ "image": {
254
+ "path": str(self.image.path) if self.image.path is not None else None,
255
+ "url": self.image.url,
256
+ "format": self.image.format,
257
+ "size_bytes": self.image.size_bytes,
258
+ "virtual_size_bytes": self.image.virtual_size_bytes,
259
+ "expected_sha": self.image.expected_sha,
260
+ },
261
+ "target": {
262
+ "path": str(self.target.path),
263
+ "exists": self.target.exists,
264
+ "is_block_device": self.target.is_block_device,
265
+ "size_bytes": self.target.size_bytes,
266
+ "mountpoints": list(self.target.mountpoints),
267
+ },
268
+ "notes": list(self.notes),
269
+ }
270
+
271
+
272
+ # ---------- I/O: probing -----------------------------------------------------
273
+
274
+
275
+ def probe_image(path: Path) -> ImageInfo:
276
+ """Inspect an image file on disk. Raises ``FileNotFoundError`` if missing."""
277
+ if not path.exists():
278
+ raise FileNotFoundError(f"image not found: {path}")
279
+ fmt = images.detect_format(path)
280
+ return ImageInfo(
281
+ path=path,
282
+ format=fmt,
283
+ size_bytes=path.stat().st_size,
284
+ virtual_size_bytes=_image_virtual_size(path, fmt),
285
+ )
286
+
287
+
288
+ def probe_image_url(
289
+ url: str, format_hint: str | None = None, *, expected_sha: str | None = None
290
+ ) -> ImageInfo:
291
+ """Inspect an image at an HTTP/HTTPS or ``oras://`` URL.
292
+
293
+ For http(s): HEAD request, format from URL path, size from
294
+ ``Content-Length``. For ``oras://`` refs: resolve via :mod:`withcache.oras`
295
+ to a manifest layer, format inferred from the layer's title
296
+ annotation (or ``img.gz`` default), size from the manifest's layer
297
+ size. Virtual size (what gets written to disk) can only be
298
+ determined for raw ``.img`` URLs from HEAD; compressed and qcow2
299
+ URLs return ``virtual_size_bytes = None`` because computing it
300
+ would require pulling part of the body. Validation handles
301
+ ``None`` by skipping the size-fits-target check with a note.
302
+
303
+ ``format_hint`` is the catalog entry's declared format
304
+ (``CatalogEntry.format`` or ``ImageEntry.format``). When the URL
305
+ path's filename has no recognised extension -- e.g. pixie's
306
+ ``/images/<sha>/<display-name>`` route where the trailing
307
+ segment is human text without a file extension -- URL-based
308
+ detection returns ``None`` and ``validate_plan`` rejects the
309
+ plan with "image format not recognised". The hint lets the
310
+ caller (which read the catalog and knows the format) supply
311
+ it as a fallback so the probe doesn't fail just because the
312
+ URL's decorative filename lacks an extension.
313
+
314
+ ``expected_sha`` is the catalog / server's declared content digest
315
+ (bare 64-hex or ``sha256:<hex>``) for a plain-HTTP source; it is
316
+ normalised onto ``ImageInfo.expected_sha`` and verified on the wire
317
+ at flash time. ``oras://`` refs ignore it (they resolve their own
318
+ digest from the manifest).
319
+
320
+ Raises ``FileNotFoundError`` if the server doesn't respond or
321
+ returns 4xx / 5xx for the HEAD (http) or any registry call
322
+ (oras). Raises ``ValueError`` on an unsupported scheme.
323
+ """
324
+ if oras.is_oras_url(url):
325
+ return _probe_image_url_oras(url)
326
+
327
+ import urllib.error
328
+ import urllib.parse
329
+ import urllib.request
330
+
331
+ parsed = urllib.parse.urlparse(url)
332
+ if parsed.scheme not in ("http", "https"):
333
+ raise ValueError(f"image URL must be http://, https://, or oras://: {url}")
334
+ filename = Path(parsed.path).name or "image"
335
+ fmt = images.detect_format(Path(filename))
336
+ if fmt is None:
337
+ # URL filename didn't carry a recognised extension. Fall
338
+ # back to the caller-supplied hint (catalog entry's
339
+ # ``format`` field) if any. ``validate_plan`` will still
340
+ # reject ``None`` -> caller saw an "image format not
341
+ # recognised" error when both fail.
342
+ fmt = format_hint
343
+
344
+ size_bytes = 0
345
+ virtual_size_bytes: int | None = None
346
+ req = urllib.request.Request(url, method="HEAD")
347
+ try:
348
+ with urllib.request.urlopen(req, timeout=30) as resp:
349
+ # A malformed Content-Length must fold into "unknown size"
350
+ # (size_bytes stays 0) rather than crash the probe with an
351
+ # uncaught ValueError -- mirrors the guard in
352
+ # ``pixie.web._releases._stream``.
353
+ cl = resp.headers.get("Content-Length")
354
+ try:
355
+ parsed_len = int(cl) if cl is not None else None
356
+ except ValueError:
357
+ parsed_len = None
358
+ if parsed_len is not None:
359
+ size_bytes = parsed_len
360
+ if fmt == "img":
361
+ # Raw .img: source size == virtual size.
362
+ virtual_size_bytes = size_bytes
363
+ except (urllib.error.URLError, ConnectionError, TimeoutError) as exc:
364
+ raise FileNotFoundError(f"image URL not reachable: {url} ({exc})") from exc
365
+
366
+ return ImageInfo(
367
+ path=None,
368
+ url=url,
369
+ format=fmt,
370
+ size_bytes=size_bytes,
371
+ virtual_size_bytes=virtual_size_bytes,
372
+ expected_sha=_normalize_digest(expected_sha),
373
+ )
374
+
375
+
376
+ def _normalize_digest(sha: str | None) -> str | None:
377
+ """Normalise a declared digest to ``sha256:<hex>`` (or ``None``).
378
+
379
+ Catalog ``sha256`` fields carry a bare 64-char hex string; the oras
380
+ resolver and the on-wire ``sha256sum`` both speak ``sha256:<hex>``.
381
+ Normalise so :func:`_verify_digest` compares like with like. A value
382
+ already carrying the ``sha256:`` prefix passes through unchanged.
383
+ """
384
+ if sha is None:
385
+ return None
386
+ sha = sha.strip().lower()
387
+ return sha if sha.startswith("sha256:") else f"sha256:{sha}"
388
+
389
+
390
+ def _probe_image_url_oras(url: str) -> ImageInfo:
391
+ """Probe an ``oras://`` reference by resolving it to a manifest layer.
392
+
393
+ Caller already verified the scheme. Format comes from the layer's
394
+ title annotation (e.g. ``nosi-debian-sysdev-x86_64.img.gz`` ->
395
+ ``img.gz``); falls back to ``img.gz`` if no usable title (nosi's
396
+ publishing convention and the practical default for OCI-hosted
397
+ disk images). Virtual size stays ``None`` -- determining it from
398
+ a compressed blob would require pulling the whole image.
399
+ """
400
+ try:
401
+ resolved = oras.resolve_ref(url)
402
+ except oras.OrasError as exc:
403
+ # Re-raise as FileNotFoundError so ``pixie``'s existing
404
+ # "image URL not reachable" path handles it uniformly with
405
+ # plain HTTP failures.
406
+ raise FileNotFoundError(f"oras ref not resolvable: {url} ({exc})") from exc
407
+ fmt = images.detect_format(Path(resolved.title)) if resolved.title else "img.gz"
408
+ return ImageInfo(
409
+ path=None,
410
+ url=url,
411
+ format=fmt,
412
+ size_bytes=resolved.size or 0,
413
+ # Compressed: would need to pull (part of) the body. Caller
414
+ # falls back to the "skip size-fits check" branch on None.
415
+ virtual_size_bytes=None,
416
+ )
417
+
418
+
419
+ def probe_target(path: Path) -> TargetInfo:
420
+ """Inspect a candidate target path. Never raises; returns a populated info."""
421
+ if not path.exists():
422
+ return TargetInfo(
423
+ path=path,
424
+ exists=False,
425
+ is_block_device=False,
426
+ size_bytes=None,
427
+ mountpoints=[],
428
+ )
429
+
430
+ try:
431
+ st = path.stat()
432
+ except OSError:
433
+ return TargetInfo(
434
+ path=path,
435
+ exists=True,
436
+ is_block_device=False,
437
+ size_bytes=None,
438
+ mountpoints=[],
439
+ )
440
+
441
+ is_block = stat.S_ISBLK(st.st_mode)
442
+ if not is_block:
443
+ return TargetInfo(
444
+ path=path,
445
+ exists=True,
446
+ is_block_device=False,
447
+ size_bytes=None,
448
+ mountpoints=[],
449
+ )
450
+
451
+ return TargetInfo(
452
+ path=path,
453
+ exists=True,
454
+ is_block_device=True,
455
+ size_bytes=_lsblk_target_size(path),
456
+ mountpoints=_lsblk_target_mountpoints(path),
457
+ )
458
+
459
+
460
+ # ---------- Pure plan building + validation ----------------------------------
461
+
462
+
463
+ def make_plan(image: ImageInfo, target: TargetInfo) -> FlashPlan:
464
+ """Bundle probed info into a :class:`FlashPlan`. Pure; no I/O."""
465
+ plan = FlashPlan(image=image, target=target)
466
+ if image.virtual_size_bytes is None and image.format is not None:
467
+ plan.notes.append(
468
+ "image virtual size could not be determined; size-fits-target check skipped"
469
+ )
470
+ return plan
471
+
472
+
473
+ def validate_plan(plan: FlashPlan) -> list[str]:
474
+ """Return a list of error messages describing why ``plan`` is invalid.
475
+
476
+ Empty list = the plan would be safe to execute as a real flash.
477
+ Pure; no I/O.
478
+ """
479
+ errors: list[str] = []
480
+
481
+ if plan.image.format is None:
482
+ # Specific guidance when the operator dropped a tarball on
483
+ # PIXIE_IMAGES: those wrap the actual image inside per-file TAR
484
+ # headers, and pixie's flash code is single-stream-only. A
485
+ # generic "format not recognised" would leave operators
486
+ # confused; tell them what to do.
487
+ if images.is_tarball_extension(plan.image.display):
488
+ errors.append(
489
+ f"image is a tarball, not a single-file image: "
490
+ f"{plan.image.display}. Extract first "
491
+ f"(``tar -xf {plan.image.display}``) and drop the "
492
+ f"resulting .img / .qcow2 / .img.zst / .img.xz / "
493
+ f".img.gz / .img.bz2 onto PIXIE_IMAGES."
494
+ )
495
+ else:
496
+ errors.append(
497
+ f"image format not recognised: {plan.image.display} "
498
+ f"(supported: .qcow2, .img, .img.zst, .img.xz, "
499
+ f".img.gz, .img.bz2)"
500
+ )
501
+
502
+ if not plan.target.exists:
503
+ errors.append(f"target does not exist: {plan.target.path}")
504
+ elif not plan.target.is_block_device:
505
+ errors.append(f"target is not a block device: {plan.target.path}")
506
+
507
+ # A mounted partition on the target means the disk is IN USE, so
508
+ # refuse: you can usually unmount, but the fact it was mounted means
509
+ # something was using it, and silently overwriting it is the wrong
510
+ # answer. The flasher live env must NOT auto-mount the target
511
+ # (systemd.gpt_auto=0 on the kernel cmdline); if it does, that's the
512
+ # bug to fix, not a reason to clobber the disk.
513
+ if plan.target.mountpoints:
514
+ errors.append(
515
+ f"target has mounted partitions ({', '.join(plan.target.mountpoints)}); "
516
+ "the disk is in use. Unmount it first if you really mean to overwrite it."
517
+ )
518
+
519
+ if (
520
+ plan.target.size_bytes is not None
521
+ and plan.image.virtual_size_bytes is not None
522
+ and plan.image.virtual_size_bytes > plan.target.size_bytes
523
+ ):
524
+ errors.append(
525
+ f"image ({plan.image.virtual_size_bytes} bytes) "
526
+ f"is larger than target ({plan.target.size_bytes} bytes)"
527
+ )
528
+
529
+ return errors
530
+
531
+
532
+ # ---------- Real write -------------------------------------------------------
533
+
534
+
535
+ class FlashError(RuntimeError):
536
+ """Raised when a flash-related operation cannot complete.
537
+
538
+ :class:`FlashRaceError` is a subclass for the specific case where
539
+ the target's state changed between the last successful probe and
540
+ the attempted write (it became mounted, stopped being a block
541
+ device, etc.) -- ``pixie`` surfaces that as exit code 5.
542
+ """
543
+
544
+
545
+ class FlashRaceError(FlashError):
546
+ """The target changed state between probe and write (mounted, removed, ...)."""
547
+
548
+
549
+ class FlashCancelled(FlashError):
550
+ """Raised when the operator's ``cancel`` callback returns True.
551
+
552
+ Distinct from :class:`FlashError` proper so callers (the
553
+ ``pixie`` wizard, tests) can branch on "operator-requested abort"
554
+ vs "the underlying pipeline failed". Subclassing means callers
555
+ that catch
556
+ :class:`FlashError` still handle cancellation as a failure path
557
+ if they don't care about the distinction.
558
+ """
559
+
560
+
561
+ class FlashIntegrityError(FlashError):
562
+ """The fetched bytes did not hash to the source's declared digest.
563
+
564
+ Raised after a stream finishes when an ``oras://`` reference (or a
565
+ catalog entry carrying a ``sha256``) committed to a content digest
566
+ and the ``sha256sum`` computed on the wire disagrees. Subclassing
567
+ :class:`FlashError` means existing ``except FlashError`` handlers
568
+ treat a corrupted / tampered download as a flash failure; callers
569
+ that care can branch on the integrity-specific type.
570
+ """
571
+
572
+
573
+ def _spawn_cancel_watchdog(
574
+ procs: list[subprocess.Popen[Any]],
575
+ cancel: CancelCheck | None,
576
+ *,
577
+ poll_interval: float = 0.25,
578
+ terminate_grace: float = 1.0,
579
+ ) -> threading.Thread | None:
580
+ """Spawn a daemon thread that polls ``cancel()`` and kills the
581
+ pipeline subprocesses on True.
582
+
583
+ The watchdog exits naturally when all ``procs`` have finished
584
+ (so it doesn't outlive a successful flash). On cancel: SIGTERM
585
+ each live proc, give them ``terminate_grace`` seconds to drain
586
+ cleanly, then SIGKILL anything still alive. The main pipeline
587
+ will then see non-zero exit codes / EOF on its pipes; the caller
588
+ re-checks ``cancel()`` after the pipeline returns and raises
589
+ :class:`FlashCancelled` rather than :class:`FlashError`.
590
+ """
591
+ if cancel is None:
592
+ return None
593
+
594
+ def _watch() -> None:
595
+ while True:
596
+ if all(p.poll() is not None for p in procs):
597
+ return # natural completion: nothing left to kill
598
+ if cancel():
599
+ for p in procs:
600
+ if p.poll() is None:
601
+ with contextlib.suppress(ProcessLookupError):
602
+ p.terminate()
603
+ deadline = time.monotonic() + terminate_grace
604
+ for p in procs:
605
+ remaining = max(0.0, deadline - time.monotonic())
606
+ with contextlib.suppress(subprocess.TimeoutExpired):
607
+ p.wait(timeout=remaining)
608
+ for p in procs:
609
+ if p.poll() is None:
610
+ with contextlib.suppress(ProcessLookupError):
611
+ p.kill()
612
+ return
613
+ time.sleep(poll_interval)
614
+
615
+ thread = threading.Thread(target=_watch, daemon=True)
616
+ thread.start()
617
+ return thread
618
+
619
+
620
+ def execute_plan(
621
+ plan: FlashPlan,
622
+ *,
623
+ progress: ProgressCallback | None = None,
624
+ cancel: CancelCheck | None = None,
625
+ ) -> None:
626
+ """Write ``plan.image`` to ``plan.target``.
627
+
628
+ Re-probes the target immediately before writing to catch races
629
+ (target gets mounted, swapped, or removed between the dry-run and
630
+ the real flash). Dispatches to the right write strategy based on
631
+ image format. Synchronises and re-reads the partition table on
632
+ success.
633
+
634
+ If ``progress`` is given, lifecycle :class:`FlashProgress` events
635
+ are emitted: ``started``, ``writing``, ``synced``, ``partprobed``.
636
+ On any :class:`FlashError`, a ``failed`` event is emitted with the
637
+ exception string in ``note`` and the exception re-raised.
638
+
639
+ If ``cancel`` is given (a zero-arg callable returning ``bool``), a
640
+ watchdog thread polls it ~4Hz while a URL flash is streaming. On
641
+ True, the pipeline's subprocesses (``curl`` / decompressor /
642
+ ``dd``) are SIGTERM'd with a 1s grace then SIGKILL'd; the call
643
+ then raises :class:`FlashCancelled` rather than
644
+ :class:`FlashError`. Cancel applies only to the URL flash paths
645
+ (where a slow remote can leave the operator waiting); the
646
+ local-file dispatch finishes in a few seconds and isn't worth
647
+ interrupting.
648
+
649
+ Raises :class:`FlashError` for caller-visible failures (target no
650
+ longer suitable, format unrecognised, write subprocess failed).
651
+ Raises :class:`FlashCancelled` when the operator's cancel
652
+ callback returned True.
653
+ """
654
+ _emit(progress, "started", total_bytes=plan.image.virtual_size_bytes)
655
+
656
+ try:
657
+ fresh_target = probe_target(plan.target.path)
658
+ if not fresh_target.exists or not fresh_target.is_block_device:
659
+ raise FlashRaceError(f"target is no longer a block device: {plan.target.path}")
660
+ if fresh_target.mountpoints:
661
+ # A mounted partition means the disk is IN USE -- refuse.
662
+ # Partitions + data are fine (pixie overwrites whole disks);
663
+ # only a *mount* signals something is using it. We do NOT
664
+ # auto-unmount: you usually could, but the fact it was
665
+ # mounted means it was in use. The flasher live env must not
666
+ # auto-mount the target (systemd.gpt_auto=0); a mount here
667
+ # is therefore a real one, not the live env being silly.
668
+ raise FlashRaceError(
669
+ f"target has mounted partitions ({', '.join(fresh_target.mountpoints)}); "
670
+ "the disk is in use. Unmount it first if you really mean to overwrite it."
671
+ )
672
+
673
+ fmt = plan.image.format
674
+ total_bytes = plan.image.virtual_size_bytes
675
+ _emit(progress, "writing", note=fmt or "?")
676
+ if plan.image.url is not None:
677
+ # Streaming pipeline: curl URL | (optional zstd -d) | dd -> target.
678
+ # qcow2 can't stream-convert (random-access), so it's downloaded
679
+ # to a temp file first and then handed to the existing local
680
+ # qcow2 path.
681
+ expected_sha = plan.image.expected_sha
682
+ if fmt == "img":
683
+ _flash_img_from_url(
684
+ plan.image.url,
685
+ plan.target.path,
686
+ progress=progress,
687
+ total_bytes=total_bytes,
688
+ cancel=cancel,
689
+ expected_sha=expected_sha,
690
+ )
691
+ elif fmt == "img.zst":
692
+ _flash_zst_from_url(
693
+ plan.image.url,
694
+ plan.target.path,
695
+ progress=progress,
696
+ total_bytes=total_bytes,
697
+ cancel=cancel,
698
+ expected_sha=expected_sha,
699
+ )
700
+ elif fmt == "img.xz":
701
+ _flash_xz_from_url(
702
+ plan.image.url,
703
+ plan.target.path,
704
+ progress=progress,
705
+ total_bytes=total_bytes,
706
+ cancel=cancel,
707
+ expected_sha=expected_sha,
708
+ )
709
+ elif fmt == "img.gz":
710
+ _flash_gz_from_url(
711
+ plan.image.url,
712
+ plan.target.path,
713
+ progress=progress,
714
+ total_bytes=total_bytes,
715
+ cancel=cancel,
716
+ expected_sha=expected_sha,
717
+ )
718
+ elif fmt == "img.bz2":
719
+ _flash_bz2_from_url(
720
+ plan.image.url,
721
+ plan.target.path,
722
+ progress=progress,
723
+ total_bytes=total_bytes,
724
+ cancel=cancel,
725
+ expected_sha=expected_sha,
726
+ )
727
+ elif fmt == "qcow2":
728
+ _flash_qcow2_from_url(
729
+ plan.image.url,
730
+ plan.target.path,
731
+ cancel=cancel,
732
+ expected_sha=expected_sha,
733
+ )
734
+ else:
735
+ raise FlashError(f"cannot flash image of format {fmt!r}")
736
+ else:
737
+ assert plan.image.path is not None # typer narrows; validate_plan guarantees
738
+ if fmt == "img":
739
+ _flash_img(
740
+ plan.image.path,
741
+ plan.target.path,
742
+ progress=progress,
743
+ total_bytes=total_bytes,
744
+ )
745
+ elif fmt == "img.zst":
746
+ _flash_zst(
747
+ plan.image.path,
748
+ plan.target.path,
749
+ progress=progress,
750
+ total_bytes=total_bytes,
751
+ )
752
+ elif fmt == "img.xz":
753
+ _flash_xz(
754
+ plan.image.path,
755
+ plan.target.path,
756
+ progress=progress,
757
+ total_bytes=total_bytes,
758
+ )
759
+ elif fmt == "img.gz":
760
+ _flash_gz(
761
+ plan.image.path,
762
+ plan.target.path,
763
+ progress=progress,
764
+ total_bytes=total_bytes,
765
+ )
766
+ elif fmt == "img.bz2":
767
+ _flash_bz2(
768
+ plan.image.path,
769
+ plan.target.path,
770
+ progress=progress,
771
+ total_bytes=total_bytes,
772
+ )
773
+ elif fmt == "qcow2":
774
+ _flash_qcow2(plan.image.path, plan.target.path)
775
+ else:
776
+ raise FlashError(f"cannot flash image of format {fmt!r}")
777
+
778
+ _sync_target(plan.target.path)
779
+ _emit(progress, "synced")
780
+
781
+ _partprobe_target(plan.target.path)
782
+ _emit(progress, "partprobed")
783
+ except FlashError as exc:
784
+ _emit(progress, "failed", note=str(exc))
785
+ raise
786
+
787
+
788
+ # GPT partition-type GUID of an EFI System Partition (lowercase, as
789
+ # ``lsblk`` reports it).
790
+ _ESP_TYPE_GUID = "c12a7328-f81f-11d2-ba4b-00a0c93ec93b"
791
+ # The UEFI removable-media fallback loader path. dd'd images almost
792
+ # always carry this (it's what lets a stick / disk boot without an
793
+ # NVRAM entry); backslash-separated because efibootmgr wants an EFI
794
+ # path, not a POSIX one.
795
+ _UEFI_FALLBACK_LOADER = "\\EFI\\BOOT\\BOOTX64.EFI"
796
+
797
+
798
+ def _efibootmgr(args: list[str] | None = None) -> str:
799
+ """Run ``efibootmgr`` and return stdout. Raises on non-zero exit."""
800
+ return subprocess.run(
801
+ ["efibootmgr", *(args or [])],
802
+ check=True,
803
+ capture_output=True,
804
+ text=True,
805
+ timeout=30,
806
+ ).stdout
807
+
808
+
809
+ def _boot_entries_with_label(efibootmgr_out: str, label: str) -> list[str]:
810
+ """Entry numbers (``Boot####``) whose description equals ``label``.
811
+
812
+ efibootmgr prints ``Boot0007* <label>\\t<device-path>``; we match
813
+ the label portion so reflashes can drop their own prior entries
814
+ without touching the operator's / firmware's entries.
815
+ """
816
+ nums: list[str] = []
817
+ for line in efibootmgr_out.splitlines():
818
+ m = re.match(r"^Boot([0-9A-Fa-f]{4})\*?\s+(.*)$", line)
819
+ if m and m.group(2).split("\t", 1)[0].strip() == label:
820
+ nums.append(m.group(1))
821
+ return nums
822
+
823
+
824
+ def _find_esp_partition_number(disk: Path) -> int | None:
825
+ """Return the 1-based partition number of ``disk``'s EFI System
826
+ Partition, or ``None`` if it has none. Reads ``lsblk -J``."""
827
+ try:
828
+ out = subprocess.run(
829
+ ["lsblk", "-J", "-o", "PATH,PARTTYPE,PARTN", str(disk)],
830
+ check=True,
831
+ capture_output=True,
832
+ text=True,
833
+ timeout=15,
834
+ ).stdout
835
+ data = json.loads(out)
836
+ except (FileNotFoundError, subprocess.SubprocessError, OSError, ValueError):
837
+ return None
838
+ for dev in data.get("blockdevices", []):
839
+ for child in dev.get("children") or []:
840
+ if (child.get("parttype") or "").lower() != _ESP_TYPE_GUID:
841
+ continue
842
+ partn = child.get("partn")
843
+ if partn is not None:
844
+ try:
845
+ return int(partn)
846
+ except (TypeError, ValueError):
847
+ pass
848
+ # Older lsblk has no PARTN: fall back to the trailing digits
849
+ # of the device path (/dev/sda1 -> 1, /dev/nvme0n1p1 -> 1).
850
+ m = re.search(r"(\d+)$", child.get("path") or "")
851
+ if m:
852
+ return int(m.group(1))
853
+ return None
854
+
855
+
856
+ def register_uefi_boot_entry(target_disk: Path, *, label: str = "pixie flashed") -> str:
857
+ """Register (and select for next boot) a UEFI NVRAM entry for a
858
+ freshly-flashed disk, returning a one-line human status.
859
+
860
+ A ``dd``-written image carries an EFI System Partition with a
861
+ bootloader but NO firmware boot entry (NVRAM lives in the firmware,
862
+ not the image), so a UEFI box has nothing in its boot order to boot
863
+ and falls back to netboot -- the box never comes up on its new OS.
864
+ This points the firmware at the ESP's fallback loader:
865
+
866
+ * creates an entry for ``\\EFI\\BOOT\\BOOTX64.EFI`` on the ESP,
867
+ * keeps the existing entries (the netboot entry the box just used)
868
+ FIRST in BootOrder so pixie stays in control on later boots, with
869
+ the new entry appended,
870
+ * sets BootNext to the new entry so THIS reboot lands on the disk
871
+ regardless of whether the firmware falls through on iPXE exit.
872
+
873
+ Best-effort and UEFI-only: returns a "skipped ..." status (rather
874
+ than raising) when the box isn't UEFI, ``efibootmgr`` is absent, or
875
+ the disk has no ESP. A genuine ``efibootmgr`` failure propagates as
876
+ ``CalledProcessError`` for the caller to log.
877
+ """
878
+ if not Path("/sys/firmware/efi/efivars").is_dir():
879
+ return "skipped UEFI boot entry: box is not booted in UEFI mode (BIOS/CSM)"
880
+ if shutil.which("efibootmgr") is None:
881
+ return "skipped UEFI boot entry: efibootmgr not installed in the live env"
882
+ part = _find_esp_partition_number(target_disk)
883
+ if part is None:
884
+ return f"skipped UEFI boot entry: no EFI System Partition found on {target_disk}"
885
+
886
+ # Idempotent across reflashes: drop our own prior entries first.
887
+ for num in _boot_entries_with_label(_efibootmgr(), label):
888
+ _efibootmgr(["-b", num, "-B"])
889
+
890
+ # ``--create-only`` registers the entry WITHOUT adding it to
891
+ # BootOrder. We deliberately NEVER rewrite BootOrder: doing so on
892
+ # server firmware risks dropping the box's real boot entries -- an
893
+ # earlier version that did ``-o <old_order>,<new>`` stranded an EPYC
894
+ # box out of UEFI when the old-order parse came back short. Instead
895
+ # we set only the one-shot ``BootNext``: the firmware consumes it
896
+ # after a single boot and the standing boot order is left untouched.
897
+ _efibootmgr(
898
+ [
899
+ "--create-only",
900
+ "--disk",
901
+ str(target_disk),
902
+ "--part",
903
+ str(part),
904
+ "--loader",
905
+ _UEFI_FALLBACK_LOADER,
906
+ "--label",
907
+ label,
908
+ ]
909
+ )
910
+ # We just deleted any prior pixie entries, so the entry carrying our
911
+ # label now is the one we just created.
912
+ nums = _boot_entries_with_label(_efibootmgr(), label)
913
+ if not nums:
914
+ return f"created UEFI boot entry on {target_disk} (could not confirm BootNext)"
915
+ new = nums[-1]
916
+ _efibootmgr(["-n", new]) # one-shot BootNext; BootOrder untouched
917
+ return (
918
+ f"registered UEFI boot entry Boot{new} -> {target_disk} "
919
+ f"(ESP partition {part}); BootNext set for the next boot, BootOrder untouched"
920
+ )
921
+
922
+
923
+ # Defensive scrub for subprocess stderr surfaced to the operator: an
924
+ # ``oras://`` flash injects a short-lived bearer via ``-H Authorization:
925
+ # Bearer <token>``, and curl could echo that header (e.g. on a verbose
926
+ # build or some error paths). Redact the token before it reaches the
927
+ # progress UI / logs so a captured stream can't replay the credential.
928
+ _BEARER_RE = re.compile(r"(?i)(authorization:\s*bearer\s+|bearer\s+)[A-Za-z0-9._~+/=-]+")
929
+
930
+
931
+ def _redact_secrets(line: str) -> str:
932
+ """Replace any ``Bearer <token>`` in a log line with a placeholder."""
933
+ return _BEARER_RE.sub(r"\1<redacted>", line)
934
+
935
+
936
+ def _start_subprocess_log_pump(
937
+ proc: subprocess.Popen[Any],
938
+ progress: ProgressCallback | None,
939
+ label: str,
940
+ ) -> threading.Thread | None:
941
+ """Drain ``proc.stderr`` line-by-line and emit ``subprocess_log``
942
+ events to the progress callback.
943
+
944
+ Used for auxiliary pipeline processes (zstd / gzip / xz / bzip2 /
945
+ curl) when a progress callback is set (the ``pixie`` wizard). The
946
+ wizard prints each line via ``console.print`` inside its ``with
947
+ Progress():`` context; Rich routes the line above the progress
948
+ widget without corrupting it.
949
+
950
+ Lines are decoded as UTF-8 with replacement. The reader is
951
+ newline-bound, so subprocesses that update via carriage-return-
952
+ only refresh (curl's progress bar, zstd's --no-progress=auto)
953
+ don't emit until they finally write a ``\\n`` -- exactly what we
954
+ want, since those refresh streams would otherwise spam the
955
+ progress widget.
956
+
957
+ Returns the thread (caller ``.join()``s after the proc exits) or
958
+ ``None`` if no callback is set.
959
+ """
960
+ if progress is None or proc.stderr is None:
961
+ return None
962
+
963
+ def _pump() -> None:
964
+ stream = proc.stderr
965
+ if stream is None:
966
+ return
967
+ for raw in stream:
968
+ line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
969
+ line = line.rstrip("\r\n")
970
+ if not line:
971
+ continue
972
+ _emit(progress, "subprocess_log", note=f"{label}: {_redact_secrets(line)}")
973
+
974
+ thread = threading.Thread(target=_pump, daemon=True, name=f"pixie-{label}-stderr")
975
+ thread.start()
976
+ return thread
977
+
978
+
979
+ def _start_dd_progress_thread(
980
+ proc: subprocess.Popen[str],
981
+ progress: ProgressCallback | None,
982
+ total_bytes: int | None,
983
+ *,
984
+ event: str = "writing_progress",
985
+ bytes_field: str = "bytes_written",
986
+ ) -> threading.Thread | None:
987
+ """Spawn the dd-stderr pump if a progress callback is provided.
988
+
989
+ Returns the thread (so the caller can ``.join()`` after dd exits)
990
+ or ``None`` if no callback was given. When ``progress`` is ``None``
991
+ the caller leaves dd's stderr inherited and dd's status=progress
992
+ output goes to the operator's terminal as before.
993
+
994
+ ``event`` + ``bytes_field`` are forwarded to :func:`_pump_dd_progress`
995
+ so the same helper can drive either the disk-write progress bar
996
+ (defaults) or the network-side download progress bar
997
+ (``"downloading_progress"`` / ``"bytes_downloaded"``).
998
+ """
999
+ if progress is None or proc.stderr is None:
1000
+ return None
1001
+ thread = threading.Thread(
1002
+ target=_pump_dd_progress,
1003
+ args=(proc.stderr, progress, total_bytes),
1004
+ kwargs={"event": event, "bytes_field": bytes_field},
1005
+ daemon=True,
1006
+ )
1007
+ thread.start()
1008
+ return thread
1009
+
1010
+
1011
+ def _spawn_download_meter(
1012
+ upstream_stdout: IO[bytes],
1013
+ progress: ProgressCallback | None,
1014
+ total_bytes: int | None,
1015
+ ) -> tuple[subprocess.Popen[Any] | None, threading.Thread | None]:
1016
+ """Insert a ``dd bs=1M status=progress`` between curl and the next
1017
+ pipeline stage so the operator gets a download progress bar
1018
+ distinct from the disk-write one.
1019
+
1020
+ The interposed dd just shuffles bytes through: it reads curl's
1021
+ output in 1 MiB chunks and writes them to its own stdout
1022
+ (which the caller then feeds into the next stage's stdin). The
1023
+ progress meter on its stderr is what we mine for the
1024
+ ``downloading_progress`` events; total throughput is unaffected
1025
+ in practice -- the pipe is byte-flow-bound by network speed,
1026
+ not by dd's passthrough.
1027
+
1028
+ Returns ``(proc, pump_thread)``. When ``progress`` is None the
1029
+ caller does not want a download bar; we return ``(None, None)``
1030
+ and the caller uses ``upstream_stdout`` directly.
1031
+ """
1032
+ if progress is None:
1033
+ return None, None
1034
+ proc = subprocess.Popen(
1035
+ # ``iflag=fullblock`` makes dd accumulate a full 1 MiB before
1036
+ # writing, avoiding short-read chunks that would partial-count
1037
+ # against the progress meter on a slow pipe.
1038
+ ["dd", "bs=1M", "status=progress", "iflag=fullblock"],
1039
+ stdin=upstream_stdout,
1040
+ stdout=subprocess.PIPE,
1041
+ stderr=subprocess.PIPE,
1042
+ text=True,
1043
+ )
1044
+ pump = _start_dd_progress_thread(
1045
+ proc,
1046
+ progress,
1047
+ total_bytes,
1048
+ event="downloading_progress",
1049
+ bytes_field="bytes_downloaded",
1050
+ )
1051
+ return proc, pump
1052
+
1053
+
1054
+ def _flash_img(
1055
+ image: Path,
1056
+ target: Path,
1057
+ *,
1058
+ progress: ProgressCallback | None = None,
1059
+ total_bytes: int | None = None,
1060
+ ) -> None:
1061
+ """Write a raw .img to a block device with ``dd``.
1062
+
1063
+ ``oflag=direct`` + ``conv=fsync`` are both required: O_DIRECT
1064
+ bypasses the kernel page cache so the running OS's binaries
1065
+ aren't shadowed by stale-pre-flash pages if the target happens
1066
+ to be the disk we booted from, and conv=fsync makes dd return
1067
+ only when its writes have hit disk. Combining the two is the
1068
+ only way an in-place reflash leaves a consistent on-disk state.
1069
+ """
1070
+ cmd = [
1071
+ "dd",
1072
+ f"if={image}",
1073
+ f"of={target}",
1074
+ "bs=4M",
1075
+ "oflag=direct",
1076
+ "conv=fsync",
1077
+ "status=progress",
1078
+ ]
1079
+ stderr = subprocess.PIPE if progress is not None else None
1080
+ proc = subprocess.Popen(cmd, stderr=stderr, text=True)
1081
+ pump = _start_dd_progress_thread(proc, progress, total_bytes)
1082
+ rc = proc.wait()
1083
+ if pump is not None:
1084
+ pump.join(timeout=2)
1085
+ if rc != 0:
1086
+ raise FlashError(f"dd exited {rc} writing {image} -> {target}")
1087
+
1088
+
1089
+ def _flash_compressed(
1090
+ image: Path,
1091
+ target: Path,
1092
+ decompress_cmd: list[str],
1093
+ decompress_name: str,
1094
+ *,
1095
+ progress: ProgressCallback | None = None,
1096
+ total_bytes: int | None = None,
1097
+ ) -> None:
1098
+ """Pipeline ``<decompress_cmd> | dd of=TARGET ...``.
1099
+
1100
+ Generic single-file-decompressor + dd pipeline used by every
1101
+ ``.img.<algo>`` writer. ``decompress_cmd`` reads the image
1102
+ (typically as a positional arg or via ``--stdout``-style flag)
1103
+ and writes raw decompressed bytes to its stdout, which dd
1104
+ consumes. ``decompress_name`` is used in error messages.
1105
+
1106
+ NOTE: this only handles SINGLE-FILE compression streams (zstd,
1107
+ xz, gzip, bzip2). It does NOT handle ``.tar.gz`` /
1108
+ ``.tar.xz`` / ``.zip`` containers -- those wrap one-or-many
1109
+ files inside metadata, and dd'ing a decompressed tar stream
1110
+ would write tar headers into the target's MBR. Format
1111
+ detection in ``images.py`` deliberately rejects tarball
1112
+ extensions.
1113
+ """
1114
+ # When a progress callback is set (``pixie`` wizard caller), pipe
1115
+ # the decompressor's stderr into a pump thread that emits
1116
+ # ``subprocess_log`` events; the wizard routes those through
1117
+ # Rich's console so they print above the progress widget without
1118
+ # corrupting it. Callers without a progress callback leave stderr
1119
+ # inherited (operator's tty sees zstd/gzip output natively).
1120
+ decomp_stderr = subprocess.PIPE if progress is not None else None
1121
+ decomp_proc = subprocess.Popen(decompress_cmd, stdout=subprocess.PIPE, stderr=decomp_stderr)
1122
+ decomp_log_pump = _start_subprocess_log_pump(decomp_proc, progress, decompress_name)
1123
+ try:
1124
+ stderr = subprocess.PIPE if progress is not None else None
1125
+ dd_proc = subprocess.Popen(
1126
+ [
1127
+ "dd",
1128
+ f"of={target}",
1129
+ "bs=4M",
1130
+ "oflag=direct",
1131
+ "conv=fsync",
1132
+ "status=progress",
1133
+ ],
1134
+ stdin=decomp_proc.stdout,
1135
+ stderr=stderr,
1136
+ text=True,
1137
+ )
1138
+ pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
1139
+ # Let the decompressor see SIGPIPE if dd exits early.
1140
+ if decomp_proc.stdout is not None:
1141
+ decomp_proc.stdout.close()
1142
+ dd_rc = dd_proc.wait()
1143
+ if pump is not None:
1144
+ pump.join(timeout=2)
1145
+ finally:
1146
+ decomp_rc = decomp_proc.wait()
1147
+ if decomp_log_pump is not None:
1148
+ decomp_log_pump.join(timeout=2)
1149
+
1150
+ if dd_rc != 0:
1151
+ raise FlashError(f"dd exited {dd_rc} writing {image} -> {target}")
1152
+ if decomp_rc != 0:
1153
+ raise FlashError(f"{decompress_name} exited {decomp_rc} decompressing {image}")
1154
+
1155
+
1156
+ def _flash_zst(
1157
+ image: Path,
1158
+ target: Path,
1159
+ *,
1160
+ progress: ProgressCallback | None = None,
1161
+ total_bytes: int | None = None,
1162
+ ) -> None:
1163
+ """Pipeline ``zstd -d --stdout IMG | dd of=TARGET ...``."""
1164
+ _flash_compressed(
1165
+ image,
1166
+ target,
1167
+ ["zstd", "-d", "--stdout", str(image)],
1168
+ "zstd",
1169
+ progress=progress,
1170
+ total_bytes=total_bytes,
1171
+ )
1172
+
1173
+
1174
+ def _flash_xz(
1175
+ image: Path,
1176
+ target: Path,
1177
+ *,
1178
+ progress: ProgressCallback | None = None,
1179
+ total_bytes: int | None = None,
1180
+ ) -> None:
1181
+ """Pipeline ``xz -d --stdout IMG | dd of=TARGET ...``.
1182
+
1183
+ xz decompresses at ~50-100 MB/s vs zstd's ~800-1500 MB/s;
1184
+ pixie's own target images ship as .img.zst for the per-job
1185
+ CI reflash hot path, but this writer accepts operator-supplied
1186
+ .img.xz so neither format is forced on operators.
1187
+ """
1188
+ _flash_compressed(
1189
+ image,
1190
+ target,
1191
+ ["xz", "-d", "--stdout", str(image)],
1192
+ "xz",
1193
+ progress=progress,
1194
+ total_bytes=total_bytes,
1195
+ )
1196
+
1197
+
1198
+ def _flash_gz(
1199
+ image: Path,
1200
+ target: Path,
1201
+ *,
1202
+ progress: ProgressCallback | None = None,
1203
+ total_bytes: int | None = None,
1204
+ ) -> None:
1205
+ """Pipeline ``gzip -d --stdout IMG | dd of=TARGET ...``.
1206
+
1207
+ gzip is universally available and many older distro images
1208
+ still ship as .img.gz (Raspberry Pi OS pre-2022, older
1209
+ Ubuntu Server cloud images, vendor appliance bundles).
1210
+ Decompression is fast (~300-500 MB/s) but compression ratio
1211
+ is weaker than xz/zstd on zero-heavy images.
1212
+ """
1213
+ _flash_compressed(
1214
+ image,
1215
+ target,
1216
+ ["gzip", "-d", "--stdout", str(image)],
1217
+ "gzip",
1218
+ progress=progress,
1219
+ total_bytes=total_bytes,
1220
+ )
1221
+
1222
+
1223
+ def _flash_bz2(
1224
+ image: Path,
1225
+ target: Path,
1226
+ *,
1227
+ progress: ProgressCallback | None = None,
1228
+ total_bytes: int | None = None,
1229
+ ) -> None:
1230
+ """Pipeline ``bzip2 -d --stdout IMG | dd of=TARGET ...``.
1231
+
1232
+ Decompression is the slowest of the supported formats
1233
+ (~10-30 MB/s) and bz2 lacks a metadata header for uncompressed
1234
+ size, so ``virtual_size_bytes`` is always ``None`` for .img.bz2 and
1235
+ validate_plan skips the size-fits-target check with a note.
1236
+ """
1237
+ _flash_compressed(
1238
+ image,
1239
+ target,
1240
+ ["bzip2", "-d", "--stdout", str(image)],
1241
+ "bzip2",
1242
+ progress=progress,
1243
+ total_bytes=total_bytes,
1244
+ )
1245
+
1246
+
1247
+ def _flash_qcow2(image: Path, target: Path) -> None:
1248
+ """Write a qcow2 to a block device by converting to raw in place.
1249
+
1250
+ Byte-level progress for qcow2 isn't plumbed through to the
1251
+ ``writing_progress`` event, so ``-p`` is dropped and stderr is
1252
+ captured instead: a failed convert then surfaces qemu-img's actual
1253
+ diagnostic (``Could not open ...``, permission denied, a corrupt
1254
+ -image message) in the ``FlashError`` rather than a bare exit code,
1255
+ which is what an operator needs when a block-device write fails.
1256
+ """
1257
+ cmd = ["qemu-img", "convert", "-O", "raw", str(image), str(target)]
1258
+ proc = subprocess.run(cmd, check=False, stderr=subprocess.PIPE, text=True)
1259
+ if proc.returncode != 0:
1260
+ detail = (proc.stderr or "").strip() or "no stderr captured"
1261
+ raise FlashError(
1262
+ f"qemu-img convert exited {proc.returncode} writing {image} -> {target}: {detail}"
1263
+ )
1264
+
1265
+
1266
+ # ---------- URL-streaming variants -------------------------------------------
1267
+ #
1268
+ # curl is the HTTP downloader: available on every Debian/Ubuntu/macOS
1269
+ # host the project supports, and well-instrumented for progress
1270
+ # reporting via ``--progress-bar`` to stderr. The pipelines mirror
1271
+ # the local-file flash functions but with curl on the front instead
1272
+ # of an open(file).
1273
+
1274
+
1275
+ # ``-fsSL``:
1276
+ # -f: fail on HTTP errors (4xx/5xx exit non-zero)
1277
+ # -s: silent (no progress meter, no diagnostic notes)
1278
+ # -S: but still show errors (without this, -s would also silence them)
1279
+ # -L: follow redirects
1280
+ # The ``-s`` is deliberate: curl's progress meter is carriage-return-
1281
+ # updated, which ``pixie``'s newline-bound subprocess-log pump can
1282
+ # only capture as the *initial* zero-state line (followed by silence
1283
+ # as the same line gets overwritten in place). Operators saw "all 0"
1284
+ # rows above the Rich progress bar; ``-s`` silences that, ``-S``
1285
+ # keeps real error lines flowing through.
1286
+ #
1287
+ # ``--http1.1``: force HTTP/1.1 on every streaming fetch.
1288
+ #
1289
+ # GHCR's blob CDN (pkg-containers.githubusercontent.com) and other OCI
1290
+ # registries fronted by HTTP/2-capable CDNs will RST_STREAM a
1291
+ # long-running blob transfer once the pre-signed redirect URL's TTL
1292
+ # expires, surfacing here as ``curl exited 92`` (CURLE_HTTP2_STREAM)
1293
+ # after a roughly fixed number of minutes regardless of bytes
1294
+ # transferred. Operators on pixie-usbboot reported this with
1295
+ # multi-GiB ``oras://ghcr.io/...`` images that aborted at the same
1296
+ # point every retry. HTTP/1.1 transfers are not subject to that
1297
+ # framing-layer reset, and HTTP/2 multiplexing buys us nothing for a
1298
+ # single large stream-to-dd transfer, so the cost is zero.
1299
+ #
1300
+ # NO ``--retry``: every curl invocation here streams into a running
1301
+ # ``dd`` pipeline. If curl retries on a transient network failure,
1302
+ # it re-fetches from byte 0; those bytes get written to disk a
1303
+ # SECOND time, corrupting whatever was already there. Symptom
1304
+ # observed on a Supermicro BMC flash: the Rich progress bar
1305
+ # repeatedly hit 100% then "reset" as dd kept writing past the
1306
+ # image's compressed-size total. For streaming-to-dd the right
1307
+ # behaviour is fail-fast -- the operator gets a clean error and
1308
+ # can re-flash from scratch instead of seeing a silently-corrupted
1309
+ # target. ``--retry`` would only make sense if we also passed
1310
+ # ``--continue-at`` and made dd resumable, which is a much bigger
1311
+ # refactor for a much rarer win.
1312
+ _CURL_BASE = ("curl", "-fsSL", "--http1.1")
1313
+
1314
+
1315
+ def _curl_args_for_source(url: str) -> tuple[list[str], int | None, str | None]:
1316
+ """Build curl arguments for a fetch source.
1317
+
1318
+ Plain http(s) URLs pass through unchanged. ``oras://`` references
1319
+ go through :mod:`withcache.oras` to resolve the manifest layer, and the
1320
+ resulting bearer token is injected as a ``-H Authorization``
1321
+ header on the curl call. Returns ``(argv, expected_size_or_None,
1322
+ expected_digest_or_None)`` -- the size is the manifest's declared
1323
+ layer size when known (a fallback ``total_bytes`` for callers that
1324
+ skipped HEAD), and the digest (``sha256:<hex>``) is the layer's
1325
+ content address, frozen at resolve time, for the streaming
1326
+ integrity check. Plain URLs carry neither, so both come back
1327
+ ``None`` and the caller keeps its zero-copy path.
1328
+ """
1329
+ if not oras.is_oras_url(url):
1330
+ return [*_CURL_BASE, url], None, None
1331
+ resolved = oras.resolve_ref(url)
1332
+ args = [*_CURL_BASE]
1333
+ for header_name, header_value in resolved.headers.items():
1334
+ args.extend(["-H", f"{header_name}: {header_value}"])
1335
+ args.append(resolved.blob_url)
1336
+ return args, resolved.size, resolved.digest
1337
+
1338
+
1339
+ def _spawn_hash_tee(src: IO[bytes]) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]:
1340
+ """Splice ``tee | sha256sum`` onto ``src`` (curl's read pipe).
1341
+
1342
+ Returns ``(tee_proc, sha_proc)``. ``tee_proc.stdout`` carries the
1343
+ bytes onward to the next stage (dd / decompressor); ``sha_proc``
1344
+ hashes the duplicated copy and emits ``<hex> -`` once the stream
1345
+ drains. The hash runs entirely in subprocesses -- payload bytes
1346
+ never pass through the Python process. ``tee`` writes to
1347
+ ``/dev/fd/N`` (N = the sha pipe's write end, kept open across the
1348
+ fork via ``pass_fds``); the parent's copy of that write end is
1349
+ dropped here so ``sha256sum`` sees EOF when ``tee`` exits.
1350
+ """
1351
+ sha_proc = subprocess.Popen(["sha256sum"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
1352
+ assert sha_proc.stdin is not None
1353
+ sha_fd = sha_proc.stdin.fileno()
1354
+ tee_proc = subprocess.Popen(
1355
+ ["tee", f"/dev/fd/{sha_fd}"],
1356
+ stdin=src,
1357
+ stdout=subprocess.PIPE,
1358
+ pass_fds=(sha_fd,),
1359
+ )
1360
+ sha_proc.stdin.close()
1361
+ return tee_proc, sha_proc
1362
+
1363
+
1364
+ def _read_observed_digest(sha_proc: subprocess.Popen[bytes]) -> str:
1365
+ """Read the ``sha256:<hex>`` digest from a finished ``sha256sum``.
1366
+
1367
+ ``sha256sum`` emits ``<hex> -`` once; its output is ~65 bytes so
1368
+ it fits the pipe buffer and never blocks the upstream pipeline.
1369
+
1370
+ Reads ``stdout`` directly rather than via ``communicate()``:
1371
+ :func:`_spawn_hash_tee` already closed the parent's copy of
1372
+ ``stdin`` (to hand sha256sum its EOF), and ``communicate()`` would
1373
+ try to flush that closed handle and raise ``ValueError``.
1374
+ """
1375
+ assert sha_proc.stdout is not None
1376
+ out: bytes = sha_proc.stdout.read()
1377
+ sha_proc.stdout.close()
1378
+ return "sha256:" + out.split()[0].decode()
1379
+
1380
+
1381
+ def _sha256_file(path: Path) -> str:
1382
+ """Hash a local file via ``sha256sum``; return ``sha256:<hex>``.
1383
+
1384
+ Used by the qcow2-from-URL path, which lands the download on disk
1385
+ before conversion. Shelling out keeps the bytes out of Python, the
1386
+ same plane as the streaming ``tee | sha256sum`` splice.
1387
+ """
1388
+ out = subprocess.run(
1389
+ ["sha256sum", str(path)], capture_output=True, text=True, check=True
1390
+ ).stdout
1391
+ return "sha256:" + out.split()[0]
1392
+
1393
+
1394
+ def _verify_digest(expected: str, observed: str | None, url: str) -> None:
1395
+ """Raise :class:`FlashIntegrityError` if ``observed`` != ``expected``.
1396
+
1397
+ Verification happens after ``dd`` has already written (a stream
1398
+ can't be checked before it's written), so the message warns that
1399
+ the target now holds unverified bytes and must be re-flashed.
1400
+ """
1401
+ if observed is not None and observed != expected:
1402
+ raise FlashIntegrityError(
1403
+ f"integrity check failed for {url}: expected {expected}, computed {observed}. "
1404
+ "The target now holds unverified data; re-flash from a trusted source."
1405
+ )
1406
+
1407
+
1408
+ def _flash_img_from_url(
1409
+ url: str,
1410
+ target: Path,
1411
+ *,
1412
+ progress: ProgressCallback | None = None,
1413
+ total_bytes: int | None = None,
1414
+ cancel: CancelCheck | None = None,
1415
+ expected_sha: str | None = None,
1416
+ ) -> None:
1417
+ """Stream a raw .img from URL straight to a block device with dd."""
1418
+ curl_args, resolved_size, oras_digest = _curl_args_for_source(url)
1419
+ # oras refs resolve their own layer digest; a plain-HTTP source
1420
+ # carries the catalog's declared sha instead. Either gates the
1421
+ # tee | sha256sum splice and the post-write verification.
1422
+ digest = oras_digest or expected_sha
1423
+ if total_bytes is None:
1424
+ total_bytes = resolved_size
1425
+ # Pipe curl's stderr through the subprocess-log pump so ``pixie``
1426
+ # can surface curl's lines (errors + final status) above its
1427
+ # progress widget. curl's live progress bar uses ``\r``-only
1428
+ # refresh which the newline-bound pump intentionally skips; the
1429
+ # operator sees errors + the end-of-run line, not the noisy
1430
+ # real-time bar.
1431
+ curl_stderr = subprocess.PIPE if progress is not None else None
1432
+ curl_proc = subprocess.Popen(curl_args, stdout=subprocess.PIPE, stderr=curl_stderr)
1433
+ curl_log_pump = _start_subprocess_log_pump(curl_proc, progress, "curl")
1434
+ # Interpose a ``dd`` between curl and the next stage so the
1435
+ # network-side throughput surfaces as a separate progress bar.
1436
+ # ``resolved_size`` (curl's Content-Length / oras layer size) is
1437
+ # the network total even when the decompressed payload is larger.
1438
+ assert curl_proc.stdout is not None
1439
+ dl_proc, dl_pump = _spawn_download_meter(curl_proc.stdout, progress, resolved_size)
1440
+ tee_proc: subprocess.Popen[bytes] | None = None
1441
+ sha_proc: subprocess.Popen[bytes] | None = None
1442
+ observed: str | None = None
1443
+ try:
1444
+ # Bytes go curl -> (dl_proc, when progress is on) -> [tee | sha256sum] -> dd.
1445
+ downstream_source = dl_proc.stdout if dl_proc is not None else curl_proc.stdout
1446
+ assert downstream_source is not None
1447
+ if digest is not None:
1448
+ tee_proc, sha_proc = _spawn_hash_tee(downstream_source)
1449
+ dd_stdin = tee_proc.stdout
1450
+ else:
1451
+ dd_stdin = downstream_source
1452
+ stderr = subprocess.PIPE if progress is not None else None
1453
+ dd_proc = subprocess.Popen(
1454
+ ["dd", f"of={target}", "bs=4M", "oflag=direct", "conv=fsync", "status=progress"],
1455
+ stdin=dd_stdin,
1456
+ stderr=stderr,
1457
+ text=True,
1458
+ )
1459
+ procs: list[subprocess.Popen[Any]] = [curl_proc, dd_proc]
1460
+ if dl_proc is not None:
1461
+ procs.append(dl_proc)
1462
+ if tee_proc is not None and sha_proc is not None:
1463
+ procs += [tee_proc, sha_proc]
1464
+ watchdog = _spawn_cancel_watchdog(procs, cancel)
1465
+ pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
1466
+ # Hand the read ends fully to their consumers; closing our copies
1467
+ # lets the kernel propagate EOF / SIGPIPE when one end finishes.
1468
+ if curl_proc.stdout is not None:
1469
+ curl_proc.stdout.close()
1470
+ if dl_proc is not None and dl_proc.stdout is not None:
1471
+ dl_proc.stdout.close()
1472
+ if tee_proc is not None and tee_proc.stdout is not None:
1473
+ tee_proc.stdout.close()
1474
+ dd_rc = dd_proc.wait()
1475
+ if sha_proc is not None:
1476
+ observed = _read_observed_digest(sha_proc)
1477
+ if pump is not None:
1478
+ pump.join(timeout=2)
1479
+ if dl_pump is not None:
1480
+ dl_pump.join(timeout=2)
1481
+ if watchdog is not None:
1482
+ watchdog.join(timeout=2)
1483
+ finally:
1484
+ curl_rc = curl_proc.wait()
1485
+ dl_rc = dl_proc.wait() if dl_proc is not None else 0
1486
+ tee_rc = tee_proc.wait() if tee_proc is not None else 0
1487
+ sha_rc = sha_proc.wait() if sha_proc is not None else 0
1488
+ if curl_log_pump is not None:
1489
+ curl_log_pump.join(timeout=2)
1490
+ # Cancel takes precedence over non-zero exit codes: SIGTERM
1491
+ # leaves curl/dd with nonzero status which would otherwise be
1492
+ # mis-reported as a transport failure.
1493
+ if cancel is not None and cancel():
1494
+ raise FlashCancelled("flash cancelled by operator")
1495
+ if curl_rc != 0:
1496
+ raise FlashError(f"curl exited {curl_rc} fetching {url}")
1497
+ if dl_rc != 0:
1498
+ raise FlashError(f"download meter exited {dl_rc} fetching {url}")
1499
+ if tee_rc != 0:
1500
+ raise FlashError(f"tee exited {tee_rc} fetching {url}")
1501
+ if sha_rc != 0:
1502
+ raise FlashError(f"sha256sum exited {sha_rc} hashing {url}")
1503
+ if dd_rc != 0:
1504
+ raise FlashError(f"dd exited {dd_rc} writing {url} -> {target}")
1505
+ if digest is not None:
1506
+ _verify_digest(digest, observed, url)
1507
+
1508
+
1509
+ def _flash_compressed_from_url(
1510
+ url: str,
1511
+ target: Path,
1512
+ decompress_cmd: list[str],
1513
+ decompress_name: str,
1514
+ *,
1515
+ progress: ProgressCallback | None = None,
1516
+ total_bytes: int | None = None,
1517
+ cancel: CancelCheck | None = None,
1518
+ expected_sha: str | None = None,
1519
+ ) -> None:
1520
+ """Pipeline ``curl URL | <decompress_cmd> | dd of=TARGET ...``.
1521
+
1522
+ Generic version of the URL-streaming compressed flash path used
1523
+ by every ``.img.<algo>`` URL writer. ``decompress_cmd`` reads
1524
+ from stdin (no positional file arg).
1525
+
1526
+ Same single-file caveat as ``_flash_compressed``: tarballs and
1527
+ other multi-file containers must NOT be flashed through here.
1528
+
1529
+ Progress denominator note: ``dd`` reports OUTPUT bytes (the
1530
+ decompressed bytes written to the target disk), but the upstream
1531
+ compressed blob's size is generally smaller -- often dramatically
1532
+ so for a sparse raw image. We pass ``total_bytes`` through
1533
+ unchanged: the caller (``probe_image_url`` -> ``ImageInfo
1534
+ .virtual_size_bytes``) supplies the decompressed size when it
1535
+ can derive it, and ``None`` otherwise. We deliberately do NOT
1536
+ fall back to ``_curl_args_for_source``'s ``resolved_size``
1537
+ (the compressed blob size) here -- that mismatch makes the
1538
+ progress bar overshoot to ~6x for highly compressible .img.gz
1539
+ inputs.
1540
+ """
1541
+ curl_args, resolved_compressed_size, oras_digest = _curl_args_for_source(url)
1542
+ # The digest covers the compressed blob: oras resolves its own, a
1543
+ # plain-HTTP source carries the catalog's declared sha. Either gates
1544
+ # the integrity splice.
1545
+ digest = oras_digest or expected_sha
1546
+ # Pipe both curl + decompressor stderr through subprocess-log
1547
+ # pumps. The ``pixie`` wizard prints each line above its progress
1548
+ # widget via Rich's ``console.print`` (which Rich routes around
1549
+ # the live display). Newline-bound reads mean curl's/zstd's CR-only
1550
+ # real-time refresh doesn't fire; only meaningful lines do.
1551
+ pipeline_stderr = subprocess.PIPE if progress is not None else None
1552
+ curl_proc = subprocess.Popen(curl_args, stdout=subprocess.PIPE, stderr=pipeline_stderr)
1553
+ curl_log_pump = _start_subprocess_log_pump(curl_proc, progress, "curl")
1554
+ # Insert a download meter so the operator sees network throughput
1555
+ # separately from the disk-write throughput (especially valuable
1556
+ # here: for compressed images the two numbers differ by the
1557
+ # compression ratio).
1558
+ assert curl_proc.stdout is not None
1559
+ dl_proc, dl_pump = _spawn_download_meter(curl_proc.stdout, progress, resolved_compressed_size)
1560
+ tee_proc: subprocess.Popen[bytes] | None = None
1561
+ sha_proc: subprocess.Popen[bytes] | None = None
1562
+ observed: str | None = None
1563
+ try:
1564
+ # The digest covers the COMPRESSED blob (the oras layer), so
1565
+ # the tee splices between curl (via the download meter, when
1566
+ # active) and the decompressor; it hashes the bytes curl
1567
+ # fetched, not the expanded image.
1568
+ downstream_source = dl_proc.stdout if dl_proc is not None else curl_proc.stdout
1569
+ assert downstream_source is not None
1570
+ if digest is not None:
1571
+ tee_proc, sha_proc = _spawn_hash_tee(downstream_source)
1572
+ decomp_stdin = tee_proc.stdout
1573
+ else:
1574
+ decomp_stdin = downstream_source
1575
+ decomp_proc = subprocess.Popen(
1576
+ decompress_cmd,
1577
+ stdin=decomp_stdin,
1578
+ stdout=subprocess.PIPE,
1579
+ stderr=pipeline_stderr,
1580
+ )
1581
+ decomp_log_pump = _start_subprocess_log_pump(decomp_proc, progress, decompress_name)
1582
+ if curl_proc.stdout is not None:
1583
+ curl_proc.stdout.close()
1584
+ if dl_proc is not None and dl_proc.stdout is not None:
1585
+ dl_proc.stdout.close()
1586
+ if tee_proc is not None and tee_proc.stdout is not None:
1587
+ tee_proc.stdout.close()
1588
+ try:
1589
+ stderr = subprocess.PIPE if progress is not None else None
1590
+ dd_proc = subprocess.Popen(
1591
+ ["dd", f"of={target}", "bs=4M", "oflag=direct", "conv=fsync", "status=progress"],
1592
+ stdin=decomp_proc.stdout,
1593
+ stderr=stderr,
1594
+ text=True,
1595
+ )
1596
+ procs: list[subprocess.Popen[Any]] = [curl_proc, decomp_proc, dd_proc]
1597
+ if dl_proc is not None:
1598
+ procs.append(dl_proc)
1599
+ if tee_proc is not None and sha_proc is not None:
1600
+ procs += [tee_proc, sha_proc]
1601
+ watchdog = _spawn_cancel_watchdog(procs, cancel)
1602
+ pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
1603
+ if decomp_proc.stdout is not None:
1604
+ decomp_proc.stdout.close()
1605
+ dd_rc = dd_proc.wait()
1606
+ if sha_proc is not None:
1607
+ observed = _read_observed_digest(sha_proc)
1608
+ if pump is not None:
1609
+ pump.join(timeout=2)
1610
+ if dl_pump is not None:
1611
+ dl_pump.join(timeout=2)
1612
+ if watchdog is not None:
1613
+ watchdog.join(timeout=2)
1614
+ finally:
1615
+ decomp_rc = decomp_proc.wait()
1616
+ if decomp_log_pump is not None:
1617
+ decomp_log_pump.join(timeout=2)
1618
+ finally:
1619
+ curl_rc = curl_proc.wait()
1620
+ tee_rc = tee_proc.wait() if tee_proc is not None else 0
1621
+ sha_rc = sha_proc.wait() if sha_proc is not None else 0
1622
+ if curl_log_pump is not None:
1623
+ curl_log_pump.join(timeout=2)
1624
+ # Cancel takes precedence over non-zero exit codes: the SIGTERM
1625
+ # the watchdog sends leaves all three subprocesses with nonzero
1626
+ # status, which would otherwise be misread as a transport /
1627
+ # decode failure.
1628
+ if cancel is not None and cancel():
1629
+ raise FlashCancelled("flash cancelled by operator")
1630
+ if curl_rc != 0:
1631
+ raise FlashError(f"curl exited {curl_rc} fetching {url}")
1632
+ if tee_rc != 0:
1633
+ raise FlashError(f"tee exited {tee_rc} fetching {url}")
1634
+ if sha_rc != 0:
1635
+ raise FlashError(f"sha256sum exited {sha_rc} hashing {url}")
1636
+ if decomp_rc != 0:
1637
+ raise FlashError(f"{decompress_name} -d exited {decomp_rc} decompressing {url}")
1638
+ if dd_rc != 0:
1639
+ raise FlashError(f"dd exited {dd_rc} writing {url} -> {target}")
1640
+ if digest is not None:
1641
+ _verify_digest(digest, observed, url)
1642
+
1643
+
1644
+ def _flash_zst_from_url(
1645
+ url: str,
1646
+ target: Path,
1647
+ *,
1648
+ progress: ProgressCallback | None = None,
1649
+ total_bytes: int | None = None,
1650
+ cancel: CancelCheck | None = None,
1651
+ expected_sha: str | None = None,
1652
+ ) -> None:
1653
+ """Pipeline ``curl URL | zstd -d --stdout | dd of=TARGET ...``.
1654
+
1655
+ Decompresses on the fly. The compressed bytes never land on
1656
+ the local filesystem; only the raw image touches the target
1657
+ disk. This is the path the PXE-driven ``pixie`` flow uses by
1658
+ default since pixie's own target images ship as ``.img.zst``.
1659
+ """
1660
+ _flash_compressed_from_url(
1661
+ url,
1662
+ target,
1663
+ ["zstd", "-d", "--stdout"],
1664
+ "zstd",
1665
+ progress=progress,
1666
+ total_bytes=total_bytes,
1667
+ cancel=cancel,
1668
+ expected_sha=expected_sha,
1669
+ )
1670
+
1671
+
1672
+ def _flash_xz_from_url(
1673
+ url: str,
1674
+ target: Path,
1675
+ *,
1676
+ progress: ProgressCallback | None = None,
1677
+ total_bytes: int | None = None,
1678
+ cancel: CancelCheck | None = None,
1679
+ expected_sha: str | None = None,
1680
+ ) -> None:
1681
+ """Pipeline ``curl URL | xz -d --stdout | dd of=TARGET ...``."""
1682
+ _flash_compressed_from_url(
1683
+ url,
1684
+ target,
1685
+ ["xz", "-d", "--stdout"],
1686
+ "xz",
1687
+ progress=progress,
1688
+ total_bytes=total_bytes,
1689
+ cancel=cancel,
1690
+ expected_sha=expected_sha,
1691
+ )
1692
+
1693
+
1694
+ def _flash_gz_from_url(
1695
+ url: str,
1696
+ target: Path,
1697
+ *,
1698
+ progress: ProgressCallback | None = None,
1699
+ total_bytes: int | None = None,
1700
+ cancel: CancelCheck | None = None,
1701
+ expected_sha: str | None = None,
1702
+ ) -> None:
1703
+ """Pipeline ``curl URL | gzip -d --stdout | dd of=TARGET ...``."""
1704
+ _flash_compressed_from_url(
1705
+ url,
1706
+ target,
1707
+ ["gzip", "-d", "--stdout"],
1708
+ "gzip",
1709
+ progress=progress,
1710
+ total_bytes=total_bytes,
1711
+ cancel=cancel,
1712
+ expected_sha=expected_sha,
1713
+ )
1714
+
1715
+
1716
+ def _flash_bz2_from_url(
1717
+ url: str,
1718
+ target: Path,
1719
+ *,
1720
+ progress: ProgressCallback | None = None,
1721
+ total_bytes: int | None = None,
1722
+ cancel: CancelCheck | None = None,
1723
+ expected_sha: str | None = None,
1724
+ ) -> None:
1725
+ """Pipeline ``curl URL | bzip2 -d --stdout | dd of=TARGET ...``."""
1726
+ _flash_compressed_from_url(
1727
+ url,
1728
+ target,
1729
+ ["bzip2", "-d", "--stdout"],
1730
+ "bzip2",
1731
+ progress=progress,
1732
+ total_bytes=total_bytes,
1733
+ cancel=cancel,
1734
+ expected_sha=expected_sha,
1735
+ )
1736
+
1737
+
1738
+ def _flash_qcow2_from_url(
1739
+ url: str,
1740
+ target: Path,
1741
+ *,
1742
+ cancel: CancelCheck | None = None,
1743
+ expected_sha: str | None = None,
1744
+ ) -> None:
1745
+ """Download a qcow2 to a temp file, then ``qemu-img convert`` it.
1746
+
1747
+ qcow2 is random-access (the converter seeks all over the source),
1748
+ so it cannot stream. We download the whole file to a temp location
1749
+ first and reuse the existing local-qcow2 flash path.
1750
+ """
1751
+ import tempfile
1752
+
1753
+ with tempfile.NamedTemporaryFile(suffix=".qcow2", delete=False) as tmp:
1754
+ tmp_path = Path(tmp.name)
1755
+ try:
1756
+ curl_args, _, oras_digest = _curl_args_for_source(url)
1757
+ digest = oras_digest or expected_sha
1758
+ curl_argv = [*curl_args[:-1], "--output", str(tmp_path), curl_args[-1]]
1759
+ # Popen + watchdog (rather than subprocess.run) so the cancel
1760
+ # callback can terminate the download mid-stream. qcow2 can't
1761
+ # stream-flash, so the download phase is the bulk of the wall
1762
+ # time; cancelling there is what matters most for the
1763
+ # operator experience.
1764
+ curl_proc = subprocess.Popen(curl_argv)
1765
+ watchdog = _spawn_cancel_watchdog([curl_proc], cancel)
1766
+ rc = curl_proc.wait()
1767
+ if watchdog is not None:
1768
+ watchdog.join(timeout=2)
1769
+ if cancel is not None and cancel():
1770
+ raise FlashCancelled("flash cancelled by operator")
1771
+ if rc != 0:
1772
+ raise FlashError(f"curl exited {rc} fetching {url}")
1773
+ # qcow2 lands on disk before conversion (it can't stream), so the
1774
+ # integrity check hashes the temp file rather than tee-ing a pipe.
1775
+ if digest is not None:
1776
+ _verify_digest(digest, _sha256_file(tmp_path), url)
1777
+ _flash_qcow2(tmp_path, target)
1778
+ finally:
1779
+ with contextlib.suppress(FileNotFoundError):
1780
+ tmp_path.unlink()
1781
+
1782
+
1783
+ def _sync_target(target: Path) -> None:
1784
+ """Flush kernel buffers; ``target`` accepted for symmetry with the partprobe sibling."""
1785
+ del target # informational only at this stage
1786
+ # Bounded + best-effort: on a failing target disk ``sync`` can block
1787
+ # in D-state indefinitely, which would wedge flash completion AFTER
1788
+ # the bytes are already written. Bounding it matches the "never let
1789
+ # a stuck IO subsystem hang pixie" principle in _probe_run / pixie.disks;
1790
+ # a timeout is as ignorable as the non-zero exit ``check=False``
1791
+ # already swallows.
1792
+ with contextlib.suppress(subprocess.TimeoutExpired):
1793
+ subprocess.run(["sync"], check=False, timeout=30)
1794
+
1795
+
1796
+ def _partprobe_target(target: Path) -> None:
1797
+ """Ask the kernel to re-read ``target``'s partition table, then settle udev.
1798
+
1799
+ ``udevadm settle`` is run after ``partprobe`` so subsequent ``lsblk``
1800
+ queries see the new partition tree. Without it, an immediate
1801
+ follow-up (e.g. an external tool looking at partition labels)
1802
+ can race the kernel's partition scan and find no children.
1803
+
1804
+ Both calls are bounded + best-effort (see :func:`_sync_target`): a
1805
+ stuck disk must not hang the post-flash housekeeping.
1806
+ """
1807
+ with contextlib.suppress(subprocess.TimeoutExpired):
1808
+ subprocess.run(["partprobe", str(target)], check=False, timeout=30)
1809
+ with contextlib.suppress(subprocess.TimeoutExpired):
1810
+ subprocess.run(["udevadm", "settle"], check=False, timeout=30)
1811
+
1812
+
1813
+ # ---------- Internal helpers --------------------------------------------------
1814
+
1815
+
1816
+ def _probe_run(cmd: list[str], *, timeout: float = 60.0) -> subprocess.CompletedProcess[str] | None:
1817
+ """Run a bounded metadata-probe command; return the completed
1818
+ process, or ``None`` if it timed out.
1819
+
1820
+ The size / mountpoint probes here run during ``validate_plan``;
1821
+ bounding them keeps a stuck IO subsystem (failing disk, slow
1822
+ network mount, corrupt image) from wedging the pre-flash check.
1823
+ Callers already treat a failed probe as "unknown" (returning
1824
+ ``None`` / ``[]``), so a timeout folds cleanly into that best-
1825
+ effort contract. Mirrors the defensive timeouts in
1826
+ :mod:`pixie.disks` and :func:`pixie.images.inspect_image`.
1827
+ """
1828
+ try:
1829
+ return subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=timeout)
1830
+ except subprocess.TimeoutExpired:
1831
+ return None
1832
+
1833
+
1834
+ def _image_virtual_size(path: Path, image_format: str | None) -> int | None:
1835
+ """Return the byte count an image would expand to on disk."""
1836
+ if image_format == "img":
1837
+ return path.stat().st_size
1838
+
1839
+ if image_format == "qcow2":
1840
+ proc = _probe_run(["qemu-img", "info", "--output=json", str(path)])
1841
+ if proc is None or proc.returncode != 0:
1842
+ return None
1843
+ try:
1844
+ payload = json.loads(proc.stdout)
1845
+ except json.JSONDecodeError:
1846
+ return None
1847
+ size = payload.get("virtual-size")
1848
+ return size if isinstance(size, int) else None
1849
+
1850
+ if image_format == "img.zst":
1851
+ proc = _probe_run(["zstd", "-l", "--no-progress", str(path)])
1852
+ if proc is None or proc.returncode != 0:
1853
+ return None
1854
+ return _parse_compressed_listing(proc.stdout, header_prefix="Frames")
1855
+
1856
+ if image_format == "img.xz":
1857
+ proc = _probe_run(["xz", "-l", str(path)])
1858
+ if proc is None or proc.returncode != 0:
1859
+ return None
1860
+ # ``xz -l`` shares the ``Compressed Uncompressed`` two-cell
1861
+ # column layout with ``zstd -l``, so the same parser works.
1862
+ # Header line for xz is ``Strms Blocks Compressed Uncompressed
1863
+ # Ratio Check Filename``.
1864
+ return _parse_compressed_listing(proc.stdout, header_prefix="Strms")
1865
+
1866
+ if image_format == "img.gz":
1867
+ # ``gzip -l`` (a.k.a. ``gunzip -l``) emits unit-less byte
1868
+ # counts in two columns: ``compressed uncompressed ratio
1869
+ # name``. Note: gzip stores the uncompressed size mod 4 GiB
1870
+ # in the trailer, so for files >= 4 GiB the reported size
1871
+ # wraps and is wrong. validate_plan treats the result as a
1872
+ # best-effort hint; if wrong the size-fits-target check
1873
+ # might miss but the actual flash still proceeds correctly
1874
+ # since dd reads the real stream.
1875
+ proc = _probe_run(["gzip", "-l", str(path)])
1876
+ if proc is None or proc.returncode != 0:
1877
+ return None
1878
+ return _parse_gzip_listing(proc.stdout)
1879
+
1880
+ if image_format == "img.bz2":
1881
+ # bzip2 stores no uncompressed size header. Returning None
1882
+ # tells validate_plan to skip the size-fits-target check
1883
+ # with a note; flash itself proceeds normally.
1884
+ return None
1885
+
1886
+ return None
1887
+
1888
+
1889
+ def _parse_compressed_listing(listing: str, *, header_prefix: str) -> int | None:
1890
+ """Best-effort extraction of the uncompressed size from
1891
+ ``zstd -l`` or ``xz -l`` output.
1892
+
1893
+ Both tools emit a header line (``Frames Skips Compressed Uncompressed
1894
+ ...`` for zstd, ``Strms Blocks Compressed Uncompressed ...`` for xz)
1895
+ followed by a row whose 2nd ``<value> <unit>`` pair is the
1896
+ uncompressed size. ``header_prefix`` selects which header line
1897
+ to skip when scanning for the data row.
1898
+ """
1899
+ for line in listing.splitlines():
1900
+ if not line.strip() or line.lstrip().startswith((header_prefix, "-")):
1901
+ continue
1902
+ cells = _ZSTD_SIZE_RE.findall(line)
1903
+ if len(cells) >= 2:
1904
+ value_str, unit = cells[1]
1905
+ try:
1906
+ value = float(value_str)
1907
+ except ValueError:
1908
+ return None
1909
+ multiplier = _ZSTD_SIZE_UNITS.get(unit)
1910
+ return int(value * multiplier) if multiplier is not None else None
1911
+ return None
1912
+
1913
+
1914
+ def _parse_gzip_listing(gzip_output: str) -> int | None:
1915
+ """Best-effort uncompressed-size extraction from ``gzip -l`` output.
1916
+
1917
+ Output shape (no units, just decimal bytes):
1918
+
1919
+ compressed uncompressed ratio uncompressed_name
1920
+ 73 37 -34.4% file
1921
+
1922
+ Skips the header line and any lines that don't have at least
1923
+ two integer columns. Returns the second integer column.
1924
+ Returns ``None`` if parsing fails OR if the reported uncompressed
1925
+ size has wrapped past 2^32 (gzip stores uncompressed size mod
1926
+ 4 GiB in the file trailer): when the reported uncompressed value
1927
+ is smaller than the compressed value, the wrap definitely
1928
+ happened, the number is a lie, and ``validate_plan`` would be
1929
+ fooled into thinking a too-big image fits a too-small disk. A
1930
+ None return tells ``validate_plan`` to skip the size-fits check
1931
+ with a note instead of trusting a wrapped value.
1932
+ """
1933
+ for line in gzip_output.splitlines():
1934
+ stripped = line.strip()
1935
+ if not stripped or stripped.startswith(("compressed", "-")):
1936
+ continue
1937
+ cells = stripped.split()
1938
+ if len(cells) < 2:
1939
+ continue
1940
+ try:
1941
+ compressed = int(cells[0])
1942
+ uncompressed = int(cells[1])
1943
+ except ValueError:
1944
+ continue
1945
+ if uncompressed < compressed:
1946
+ # 4 GiB wrap: the trailer's stored size is smaller than
1947
+ # the on-disk compressed bytes. Refuse the lie.
1948
+ return None
1949
+ return uncompressed
1950
+ return None
1951
+
1952
+
1953
+ def _lsblk_target_size(target: Path) -> int | None:
1954
+ """Return target size in bytes via ``lsblk -bndo SIZE`` (top-level only)."""
1955
+ proc = _probe_run(["lsblk", "-bndo", "SIZE", str(target)])
1956
+ if proc is None or proc.returncode != 0:
1957
+ return None
1958
+ stdout = proc.stdout.strip()
1959
+ line = stdout.splitlines()[0] if stdout else ""
1960
+ try:
1961
+ return int(line)
1962
+ except ValueError:
1963
+ return None
1964
+
1965
+
1966
+ def _lsblk_target_mountpoints(target: Path) -> list[str]:
1967
+ """Return all mountpoints used by ``target`` and its partitions."""
1968
+ proc = _probe_run(["lsblk", "-no", "MOUNTPOINTS", str(target)])
1969
+ if proc is None or proc.returncode != 0:
1970
+ return []
1971
+ return [mp for raw in proc.stdout.splitlines() if (mp := raw.strip())]