pixie-lab 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pixie/__init__.py +20 -0
- pixie/_util.py +38 -0
- pixie/catalog/__init__.py +25 -0
- pixie/catalog/_fetcher.py +513 -0
- pixie/catalog/_routes.py +392 -0
- pixie/catalog/_schema.py +263 -0
- pixie/catalog/_store.py +234 -0
- pixie/deploy/__init__.py +26 -0
- pixie/deploy/_main.py +321 -0
- pixie/deploy/_templates.py +153 -0
- pixie/disks.py +85 -0
- pixie/events/__init__.py +28 -0
- pixie/events/_kinds.py +203 -0
- pixie/events/_log.py +210 -0
- pixie/events/_routes.py +42 -0
- pixie/exports/__init__.py +17 -0
- pixie/exports/_routes.py +211 -0
- pixie/exports/_store.py +156 -0
- pixie/exports/_supervisor.py +240 -0
- pixie/flash.py +1971 -0
- pixie/images.py +473 -0
- pixie/machines/__init__.py +16 -0
- pixie/machines/_routes.py +190 -0
- pixie/machines/_store.py +623 -0
- pixie/oras.py +547 -0
- pixie/pivot/__init__.py +180 -0
- pixie/pivot/nbdboot +348 -0
- pixie/pxe/__init__.py +19 -0
- pixie/pxe/_renderer.py +244 -0
- pixie/pxe/_routes.py +386 -0
- pixie/tftp/__init__.py +21 -0
- pixie/tftp/_supervisor.py +129 -0
- pixie/tui/__init__.py +185 -0
- pixie/tui/_app.py +2219 -0
- pixie/tui_catalog.py +657 -0
- pixie/web/__init__.py +6 -0
- pixie/web/_auth.py +70 -0
- pixie/web/_settings_store.py +211 -0
- pixie/web/_static/.gitkeep +0 -0
- pixie/web/_static/bootstrap-icons.min.css +5 -0
- pixie/web/_static/bootstrap.min.css +12 -0
- pixie/web/_static/fonts/bootstrap-icons.woff +0 -0
- pixie/web/_static/fonts/bootstrap-icons.woff2 +0 -0
- pixie/web/_static/htmx.min.js +1 -0
- pixie/web/_static/pixie-favicon.png +0 -0
- pixie/web/_static/sse.js +290 -0
- pixie/web/_table_state.py +261 -0
- pixie/web/_templates/_partials/page_description.html +22 -0
- pixie/web/_templates/_partials/recent_events.html +47 -0
- pixie/web/_templates/_partials/table_helpers.html +189 -0
- pixie/web/_templates/catalog.html +364 -0
- pixie/web/_templates/catalog_detail.html +386 -0
- pixie/web/_templates/dashboard.html +256 -0
- pixie/web/_templates/events.html +125 -0
- pixie/web/_templates/ipxe/bootstrap.j2 +12 -0
- pixie/web/_templates/ipxe/exit.j2 +10 -0
- pixie/web/_templates/ipxe/nbdboot.j2 +57 -0
- pixie/web/_templates/ipxe/pixie-live-env.j2 +55 -0
- pixie/web/_templates/ipxe/unavailable.j2 +14 -0
- pixie/web/_templates/layout.html +345 -0
- pixie/web/_templates/login.html +40 -0
- pixie/web/_templates/machine_detail.html +786 -0
- pixie/web/_templates/machines.html +186 -0
- pixie/web/_templates/settings.html +265 -0
- pixie/web/main.py +1910 -0
- pixie_lab-0.1.0.dist-info/METADATA +107 -0
- pixie_lab-0.1.0.dist-info/RECORD +70 -0
- pixie_lab-0.1.0.dist-info/WHEEL +4 -0
- pixie_lab-0.1.0.dist-info/entry_points.txt +3 -0
- pixie_lab-0.1.0.dist-info/licenses/LICENSE +674 -0
pixie/flash.py
ADDED
|
@@ -0,0 +1,1971 @@
|
|
|
1
|
+
"""Flash plan: validate that an image can be written to a target disk.
|
|
2
|
+
|
|
3
|
+
Split into three layers so unit tests don't need to mock anything to
|
|
4
|
+
cover the validation logic:
|
|
5
|
+
|
|
6
|
+
- ``probe_image`` and ``probe_target`` do the I/O (reading file stats,
|
|
7
|
+
shelling out to ``qemu-img info``, ``zstd -l``, ``lsblk``) and return
|
|
8
|
+
plain :class:`ImageInfo` / :class:`TargetInfo` dataclasses.
|
|
9
|
+
- ``make_plan`` is pure: it bundles probed info into a :class:`FlashPlan`.
|
|
10
|
+
- ``validate_plan`` is pure: it returns a list of error strings.
|
|
11
|
+
- ``execute_plan`` does the destructive write (qemu-img convert /
|
|
12
|
+
zstd -d / dd as appropriate for the image format). pixie has no
|
|
13
|
+
post-flash provisioning step -- first-boot bring-up belongs in
|
|
14
|
+
the image builder (cloud-init / NoCloud); pixie only writes bytes.
|
|
15
|
+
|
|
16
|
+
The ``pixie`` wizard calls all four. Tests construct ``ImageInfo`` /
|
|
17
|
+
``TargetInfo`` directly and exercise ``make_plan`` / ``validate_plan``
|
|
18
|
+
without mocks. The probe and write functions have their own targeted
|
|
19
|
+
tests for the subprocess-shelling-out parts; integration tests against
|
|
20
|
+
a real loop device live in ``tests/test_flash_integration.py``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import contextlib
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
import shutil
|
|
29
|
+
import stat
|
|
30
|
+
import subprocess
|
|
31
|
+
import threading
|
|
32
|
+
import time
|
|
33
|
+
from collections.abc import Callable
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import IO, Any, TypeAlias
|
|
37
|
+
|
|
38
|
+
from pixie import images, oras
|
|
39
|
+
|
|
40
|
+
# ``cancel`` callbacks return True to abort an in-flight flash. The
|
|
41
|
+
# flash code polls ~4Hz from a watchdog thread; on True it terminates
|
|
42
|
+
# all child subprocesses (curl + decompressor + dd) and the main
|
|
43
|
+
# pipeline raises :class:`FlashCancelled`.
|
|
44
|
+
CancelCheck: TypeAlias = Callable[[], bool]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class FlashProgress:
|
|
49
|
+
"""One lifecycle event from :func:`execute_plan` / ``cmd_flash``.
|
|
50
|
+
|
|
51
|
+
The ``event`` field is a stable string callers dispatch on. Current
|
|
52
|
+
events:
|
|
53
|
+
|
|
54
|
+
- ``started`` - flash beginning; ``total_bytes`` is the
|
|
55
|
+
image's virtual size when known.
|
|
56
|
+
- ``writing`` - about to invoke the format-specific
|
|
57
|
+
writer (``dd`` / ``zstd | dd`` / ``qemu-img convert``).
|
|
58
|
+
- ``writing_progress`` - byte-level progress from the running
|
|
59
|
+
writer; ``bytes_written`` is set, ``total_bytes`` carries
|
|
60
|
+
through from ``started`` so consumers can compute percent /
|
|
61
|
+
ETA without holding state. Emitted ~1/sec from a daemon
|
|
62
|
+
thread that parses ``dd``'s ``status=progress`` stderr.
|
|
63
|
+
- ``downloading_progress`` - byte-level progress from the
|
|
64
|
+
network-side ``dd`` spliced between ``curl`` and the rest
|
|
65
|
+
of the URL-streaming pipeline. ``bytes_downloaded`` is set
|
|
66
|
+
and ``total_bytes`` carries the **compressed** payload size
|
|
67
|
+
(curl's Content-Length / oras layer size); compare against
|
|
68
|
+
``writing_progress``'s decompressed total to see network
|
|
69
|
+
throughput separately from disk-write throughput. Only
|
|
70
|
+
emitted on the URL-streaming flash paths
|
|
71
|
+
(``_flash_*_from_url``); local-file flashes have no
|
|
72
|
+
download stage.
|
|
73
|
+
- ``synced`` - kernel buffers flushed.
|
|
74
|
+
- ``partprobed`` - partition table re-read; flash
|
|
75
|
+
hardware-complete.
|
|
76
|
+
- ``done`` - emitted by ``cmd_flash`` after the
|
|
77
|
+
flash succeeded.
|
|
78
|
+
- ``failed`` - emitted on any :class:`FlashError`;
|
|
79
|
+
``note`` carries the exception string. The exception is then
|
|
80
|
+
re-raised.
|
|
81
|
+
- ``subprocess_log`` - one line of stderr from an auxiliary
|
|
82
|
+
pipeline subprocess (``zstd`` / ``gzip`` / ``xz`` / ``bzip2`` /
|
|
83
|
+
``curl``). ``note`` is the line, already prefixed with the
|
|
84
|
+
source label (e.g. ``"zstd: ..."``). The ``pixie`` wizard renders
|
|
85
|
+
these above its progress widget; callers without a progress
|
|
86
|
+
callback can ignore them (the subprocess's stderr is already
|
|
87
|
+
inherited in that mode). Live updates that use carriage-return-
|
|
88
|
+
only refresh (curl/zstd's own progress bars) don't show up
|
|
89
|
+
live -- the pump reads newline-terminated lines, so only the
|
|
90
|
+
final newline-terminated message lands here. That keeps the
|
|
91
|
+
Rich progress bar uncluttered while still surfacing real
|
|
92
|
+
errors + end-of-run stats.
|
|
93
|
+
|
|
94
|
+
``total_bytes`` is set on ``started`` (the image's virtual /
|
|
95
|
+
decompressed size when known) and on ``writing_progress``.
|
|
96
|
+
On ``downloading_progress`` it carries the COMPRESSED payload
|
|
97
|
+
size, which differs from the decompressed total for compressed
|
|
98
|
+
images. ``bytes_written`` is set only on ``writing_progress``;
|
|
99
|
+
``bytes_downloaded`` only on ``downloading_progress``.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
event: str
|
|
103
|
+
note: str = ""
|
|
104
|
+
total_bytes: int | None = None
|
|
105
|
+
bytes_written: int | None = None
|
|
106
|
+
bytes_downloaded: int | None = None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
ProgressCallback: TypeAlias = Callable[[FlashProgress], None]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _emit(progress: ProgressCallback | None, event: str, **fields: Any) -> None:
|
|
113
|
+
"""Call ``progress`` with a :class:`FlashProgress` if one was provided."""
|
|
114
|
+
if progress is None:
|
|
115
|
+
return
|
|
116
|
+
progress(FlashProgress(event=event, **fields))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ``dd status=progress`` writes a periodic line to stderr like:
|
|
120
|
+
# 13312000 bytes (13 MB, 13 MiB) copied, 0.103 s, 129 MB/s
|
|
121
|
+
# preceded by a ``\r`` so terminals overwrite the prior line. We parse
|
|
122
|
+
# the leading byte count and emit a ``writing_progress`` event ~1/sec.
|
|
123
|
+
_DD_PROGRESS_RE = re.compile(r"^(\d+)\s+bytes\b")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _pump_dd_progress(
|
|
127
|
+
stream: IO[str],
|
|
128
|
+
progress: ProgressCallback,
|
|
129
|
+
total_bytes: int | None,
|
|
130
|
+
*,
|
|
131
|
+
event: str = "writing_progress",
|
|
132
|
+
bytes_field: str = "bytes_written",
|
|
133
|
+
) -> None:
|
|
134
|
+
"""Read ``dd``'s stderr and emit byte-level progress events.
|
|
135
|
+
|
|
136
|
+
Designed to run in a daemon thread alongside a ``dd`` process.
|
|
137
|
+
Defaults emit the disk-write ``writing_progress`` events; pass
|
|
138
|
+
``event="downloading_progress"`` + ``bytes_field="bytes_downloaded"``
|
|
139
|
+
when the dd being pumped is the network-side splice that counts
|
|
140
|
+
bytes coming out of curl.
|
|
141
|
+
|
|
142
|
+
``dd`` separates progress lines with ``\\r`` (so each line
|
|
143
|
+
overwrites the previous one in a terminal); we replace ``\\r``
|
|
144
|
+
with ``\\n`` before splitting so we get one progress line per
|
|
145
|
+
chunk regardless of terminal-style behaviour.
|
|
146
|
+
|
|
147
|
+
Returns when ``stream`` closes (i.e. dd has exited).
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def _publish(byte_count: int) -> None:
|
|
151
|
+
_emit(
|
|
152
|
+
progress,
|
|
153
|
+
event,
|
|
154
|
+
**{bytes_field: byte_count},
|
|
155
|
+
total_bytes=total_bytes,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
buf = ""
|
|
159
|
+
while True:
|
|
160
|
+
chunk = stream.read(256)
|
|
161
|
+
if not chunk:
|
|
162
|
+
# Drain whatever's left in the buffer.
|
|
163
|
+
for line in buf.replace("\r", "\n").splitlines():
|
|
164
|
+
m = _DD_PROGRESS_RE.match(line.strip())
|
|
165
|
+
if m:
|
|
166
|
+
_publish(int(m.group(1)))
|
|
167
|
+
return
|
|
168
|
+
buf += chunk
|
|
169
|
+
# Use the LAST progress line in the buffer as the most recent
|
|
170
|
+
# snapshot. dd emits monotonically-increasing byte counts so
|
|
171
|
+
# rendering only the latest is fine.
|
|
172
|
+
lines = buf.replace("\r", "\n").splitlines()
|
|
173
|
+
if not lines:
|
|
174
|
+
continue
|
|
175
|
+
# Keep the partial trailing line for the next read.
|
|
176
|
+
if buf.endswith(("\n", "\r")):
|
|
177
|
+
buf = ""
|
|
178
|
+
else:
|
|
179
|
+
buf = lines[-1]
|
|
180
|
+
lines = lines[:-1]
|
|
181
|
+
# Find the most recent line that matches the byte-count pattern.
|
|
182
|
+
for line in reversed(lines):
|
|
183
|
+
m = _DD_PROGRESS_RE.match(line.strip())
|
|
184
|
+
if m:
|
|
185
|
+
_publish(int(m.group(1)))
|
|
186
|
+
break
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
_ZSTD_SIZE_UNITS: dict[str, int] = {
|
|
190
|
+
"B": 1,
|
|
191
|
+
"KiB": 1024,
|
|
192
|
+
"MiB": 1024**2,
|
|
193
|
+
"GiB": 1024**3,
|
|
194
|
+
"TiB": 1024**4,
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
_ZSTD_SIZE_RE = re.compile(r"([\d.]+)\s+(B|KiB|MiB|GiB|TiB)")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@dataclass
|
|
201
|
+
class ImageInfo:
|
|
202
|
+
"""Probed metadata for an image source.
|
|
203
|
+
|
|
204
|
+
Either ``path`` (a local file) or ``url`` (an HTTP/HTTPS or
|
|
205
|
+
``oras://`` reference) is set; never both. URL-sourced images
|
|
206
|
+
stream through curl directly to the target disk for ``.img`` /
|
|
207
|
+
``.img.{gz,zst,xz,bz2}`` (no temp file); for ``.qcow2`` they get
|
|
208
|
+
downloaded to a temp file first because qcow2 is random-access.
|
|
209
|
+
``oras://`` URLs go through :mod:`withcache.oras` first to resolve the
|
|
210
|
+
layer digest and inject a bearer-token Authorization header into
|
|
211
|
+
the curl call.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
path: Path | None
|
|
215
|
+
format: str | None
|
|
216
|
+
size_bytes: int
|
|
217
|
+
virtual_size_bytes: int | None # what would be written to disk; None = unknown
|
|
218
|
+
url: str | None = None
|
|
219
|
+
# Declared content digest (``sha256:<hex>``) for a URL source whose
|
|
220
|
+
# catalog / server committed to one. Verified on the wire during the
|
|
221
|
+
# flash. ``oras://`` refs resolve their own digest at flash time, so
|
|
222
|
+
# this stays ``None`` for them; it carries the catalog ``sha256`` for
|
|
223
|
+
# plain HTTP sources. ``None`` => no declared digest to check.
|
|
224
|
+
expected_sha: str | None = None
|
|
225
|
+
|
|
226
|
+
@property
|
|
227
|
+
def display(self) -> str:
|
|
228
|
+
"""User-facing identifier (URL or path string)."""
|
|
229
|
+
return self.url if self.url is not None else str(self.path)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@dataclass
|
|
233
|
+
class TargetInfo:
|
|
234
|
+
"""Probed metadata for a candidate target."""
|
|
235
|
+
|
|
236
|
+
path: Path
|
|
237
|
+
exists: bool
|
|
238
|
+
is_block_device: bool
|
|
239
|
+
size_bytes: int | None
|
|
240
|
+
mountpoints: list[str]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@dataclass
|
|
244
|
+
class FlashPlan:
|
|
245
|
+
"""Inputs and computed metadata for a flash operation."""
|
|
246
|
+
|
|
247
|
+
image: ImageInfo
|
|
248
|
+
target: TargetInfo
|
|
249
|
+
notes: list[str] = field(default_factory=list)
|
|
250
|
+
|
|
251
|
+
def to_dict(self) -> dict[str, Any]:
|
|
252
|
+
return {
|
|
253
|
+
"image": {
|
|
254
|
+
"path": str(self.image.path) if self.image.path is not None else None,
|
|
255
|
+
"url": self.image.url,
|
|
256
|
+
"format": self.image.format,
|
|
257
|
+
"size_bytes": self.image.size_bytes,
|
|
258
|
+
"virtual_size_bytes": self.image.virtual_size_bytes,
|
|
259
|
+
"expected_sha": self.image.expected_sha,
|
|
260
|
+
},
|
|
261
|
+
"target": {
|
|
262
|
+
"path": str(self.target.path),
|
|
263
|
+
"exists": self.target.exists,
|
|
264
|
+
"is_block_device": self.target.is_block_device,
|
|
265
|
+
"size_bytes": self.target.size_bytes,
|
|
266
|
+
"mountpoints": list(self.target.mountpoints),
|
|
267
|
+
},
|
|
268
|
+
"notes": list(self.notes),
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# ---------- I/O: probing -----------------------------------------------------
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def probe_image(path: Path) -> ImageInfo:
|
|
276
|
+
"""Inspect an image file on disk. Raises ``FileNotFoundError`` if missing."""
|
|
277
|
+
if not path.exists():
|
|
278
|
+
raise FileNotFoundError(f"image not found: {path}")
|
|
279
|
+
fmt = images.detect_format(path)
|
|
280
|
+
return ImageInfo(
|
|
281
|
+
path=path,
|
|
282
|
+
format=fmt,
|
|
283
|
+
size_bytes=path.stat().st_size,
|
|
284
|
+
virtual_size_bytes=_image_virtual_size(path, fmt),
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def probe_image_url(
|
|
289
|
+
url: str, format_hint: str | None = None, *, expected_sha: str | None = None
|
|
290
|
+
) -> ImageInfo:
|
|
291
|
+
"""Inspect an image at an HTTP/HTTPS or ``oras://`` URL.
|
|
292
|
+
|
|
293
|
+
For http(s): HEAD request, format from URL path, size from
|
|
294
|
+
``Content-Length``. For ``oras://`` refs: resolve via :mod:`withcache.oras`
|
|
295
|
+
to a manifest layer, format inferred from the layer's title
|
|
296
|
+
annotation (or ``img.gz`` default), size from the manifest's layer
|
|
297
|
+
size. Virtual size (what gets written to disk) can only be
|
|
298
|
+
determined for raw ``.img`` URLs from HEAD; compressed and qcow2
|
|
299
|
+
URLs return ``virtual_size_bytes = None`` because computing it
|
|
300
|
+
would require pulling part of the body. Validation handles
|
|
301
|
+
``None`` by skipping the size-fits-target check with a note.
|
|
302
|
+
|
|
303
|
+
``format_hint`` is the catalog entry's declared format
|
|
304
|
+
(``CatalogEntry.format`` or ``ImageEntry.format``). When the URL
|
|
305
|
+
path's filename has no recognised extension -- e.g. pixie's
|
|
306
|
+
``/images/<sha>/<display-name>`` route where the trailing
|
|
307
|
+
segment is human text without a file extension -- URL-based
|
|
308
|
+
detection returns ``None`` and ``validate_plan`` rejects the
|
|
309
|
+
plan with "image format not recognised". The hint lets the
|
|
310
|
+
caller (which read the catalog and knows the format) supply
|
|
311
|
+
it as a fallback so the probe doesn't fail just because the
|
|
312
|
+
URL's decorative filename lacks an extension.
|
|
313
|
+
|
|
314
|
+
``expected_sha`` is the catalog / server's declared content digest
|
|
315
|
+
(bare 64-hex or ``sha256:<hex>``) for a plain-HTTP source; it is
|
|
316
|
+
normalised onto ``ImageInfo.expected_sha`` and verified on the wire
|
|
317
|
+
at flash time. ``oras://`` refs ignore it (they resolve their own
|
|
318
|
+
digest from the manifest).
|
|
319
|
+
|
|
320
|
+
Raises ``FileNotFoundError`` if the server doesn't respond or
|
|
321
|
+
returns 4xx / 5xx for the HEAD (http) or any registry call
|
|
322
|
+
(oras). Raises ``ValueError`` on an unsupported scheme.
|
|
323
|
+
"""
|
|
324
|
+
if oras.is_oras_url(url):
|
|
325
|
+
return _probe_image_url_oras(url)
|
|
326
|
+
|
|
327
|
+
import urllib.error
|
|
328
|
+
import urllib.parse
|
|
329
|
+
import urllib.request
|
|
330
|
+
|
|
331
|
+
parsed = urllib.parse.urlparse(url)
|
|
332
|
+
if parsed.scheme not in ("http", "https"):
|
|
333
|
+
raise ValueError(f"image URL must be http://, https://, or oras://: {url}")
|
|
334
|
+
filename = Path(parsed.path).name or "image"
|
|
335
|
+
fmt = images.detect_format(Path(filename))
|
|
336
|
+
if fmt is None:
|
|
337
|
+
# URL filename didn't carry a recognised extension. Fall
|
|
338
|
+
# back to the caller-supplied hint (catalog entry's
|
|
339
|
+
# ``format`` field) if any. ``validate_plan`` will still
|
|
340
|
+
# reject ``None`` -> caller saw an "image format not
|
|
341
|
+
# recognised" error when both fail.
|
|
342
|
+
fmt = format_hint
|
|
343
|
+
|
|
344
|
+
size_bytes = 0
|
|
345
|
+
virtual_size_bytes: int | None = None
|
|
346
|
+
req = urllib.request.Request(url, method="HEAD")
|
|
347
|
+
try:
|
|
348
|
+
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
349
|
+
# A malformed Content-Length must fold into "unknown size"
|
|
350
|
+
# (size_bytes stays 0) rather than crash the probe with an
|
|
351
|
+
# uncaught ValueError -- mirrors the guard in
|
|
352
|
+
# ``pixie.web._releases._stream``.
|
|
353
|
+
cl = resp.headers.get("Content-Length")
|
|
354
|
+
try:
|
|
355
|
+
parsed_len = int(cl) if cl is not None else None
|
|
356
|
+
except ValueError:
|
|
357
|
+
parsed_len = None
|
|
358
|
+
if parsed_len is not None:
|
|
359
|
+
size_bytes = parsed_len
|
|
360
|
+
if fmt == "img":
|
|
361
|
+
# Raw .img: source size == virtual size.
|
|
362
|
+
virtual_size_bytes = size_bytes
|
|
363
|
+
except (urllib.error.URLError, ConnectionError, TimeoutError) as exc:
|
|
364
|
+
raise FileNotFoundError(f"image URL not reachable: {url} ({exc})") from exc
|
|
365
|
+
|
|
366
|
+
return ImageInfo(
|
|
367
|
+
path=None,
|
|
368
|
+
url=url,
|
|
369
|
+
format=fmt,
|
|
370
|
+
size_bytes=size_bytes,
|
|
371
|
+
virtual_size_bytes=virtual_size_bytes,
|
|
372
|
+
expected_sha=_normalize_digest(expected_sha),
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _normalize_digest(sha: str | None) -> str | None:
|
|
377
|
+
"""Normalise a declared digest to ``sha256:<hex>`` (or ``None``).
|
|
378
|
+
|
|
379
|
+
Catalog ``sha256`` fields carry a bare 64-char hex string; the oras
|
|
380
|
+
resolver and the on-wire ``sha256sum`` both speak ``sha256:<hex>``.
|
|
381
|
+
Normalise so :func:`_verify_digest` compares like with like. A value
|
|
382
|
+
already carrying the ``sha256:`` prefix passes through unchanged.
|
|
383
|
+
"""
|
|
384
|
+
if sha is None:
|
|
385
|
+
return None
|
|
386
|
+
sha = sha.strip().lower()
|
|
387
|
+
return sha if sha.startswith("sha256:") else f"sha256:{sha}"
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _probe_image_url_oras(url: str) -> ImageInfo:
|
|
391
|
+
"""Probe an ``oras://`` reference by resolving it to a manifest layer.
|
|
392
|
+
|
|
393
|
+
Caller already verified the scheme. Format comes from the layer's
|
|
394
|
+
title annotation (e.g. ``nosi-debian-sysdev-x86_64.img.gz`` ->
|
|
395
|
+
``img.gz``); falls back to ``img.gz`` if no usable title (nosi's
|
|
396
|
+
publishing convention and the practical default for OCI-hosted
|
|
397
|
+
disk images). Virtual size stays ``None`` -- determining it from
|
|
398
|
+
a compressed blob would require pulling the whole image.
|
|
399
|
+
"""
|
|
400
|
+
try:
|
|
401
|
+
resolved = oras.resolve_ref(url)
|
|
402
|
+
except oras.OrasError as exc:
|
|
403
|
+
# Re-raise as FileNotFoundError so ``pixie``'s existing
|
|
404
|
+
# "image URL not reachable" path handles it uniformly with
|
|
405
|
+
# plain HTTP failures.
|
|
406
|
+
raise FileNotFoundError(f"oras ref not resolvable: {url} ({exc})") from exc
|
|
407
|
+
fmt = images.detect_format(Path(resolved.title)) if resolved.title else "img.gz"
|
|
408
|
+
return ImageInfo(
|
|
409
|
+
path=None,
|
|
410
|
+
url=url,
|
|
411
|
+
format=fmt,
|
|
412
|
+
size_bytes=resolved.size or 0,
|
|
413
|
+
# Compressed: would need to pull (part of) the body. Caller
|
|
414
|
+
# falls back to the "skip size-fits check" branch on None.
|
|
415
|
+
virtual_size_bytes=None,
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def probe_target(path: Path) -> TargetInfo:
|
|
420
|
+
"""Inspect a candidate target path. Never raises; returns a populated info."""
|
|
421
|
+
if not path.exists():
|
|
422
|
+
return TargetInfo(
|
|
423
|
+
path=path,
|
|
424
|
+
exists=False,
|
|
425
|
+
is_block_device=False,
|
|
426
|
+
size_bytes=None,
|
|
427
|
+
mountpoints=[],
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
try:
|
|
431
|
+
st = path.stat()
|
|
432
|
+
except OSError:
|
|
433
|
+
return TargetInfo(
|
|
434
|
+
path=path,
|
|
435
|
+
exists=True,
|
|
436
|
+
is_block_device=False,
|
|
437
|
+
size_bytes=None,
|
|
438
|
+
mountpoints=[],
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
is_block = stat.S_ISBLK(st.st_mode)
|
|
442
|
+
if not is_block:
|
|
443
|
+
return TargetInfo(
|
|
444
|
+
path=path,
|
|
445
|
+
exists=True,
|
|
446
|
+
is_block_device=False,
|
|
447
|
+
size_bytes=None,
|
|
448
|
+
mountpoints=[],
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
return TargetInfo(
|
|
452
|
+
path=path,
|
|
453
|
+
exists=True,
|
|
454
|
+
is_block_device=True,
|
|
455
|
+
size_bytes=_lsblk_target_size(path),
|
|
456
|
+
mountpoints=_lsblk_target_mountpoints(path),
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
# ---------- Pure plan building + validation ----------------------------------
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def make_plan(image: ImageInfo, target: TargetInfo) -> FlashPlan:
|
|
464
|
+
"""Bundle probed info into a :class:`FlashPlan`. Pure; no I/O."""
|
|
465
|
+
plan = FlashPlan(image=image, target=target)
|
|
466
|
+
if image.virtual_size_bytes is None and image.format is not None:
|
|
467
|
+
plan.notes.append(
|
|
468
|
+
"image virtual size could not be determined; size-fits-target check skipped"
|
|
469
|
+
)
|
|
470
|
+
return plan
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def validate_plan(plan: FlashPlan) -> list[str]:
|
|
474
|
+
"""Return a list of error messages describing why ``plan`` is invalid.
|
|
475
|
+
|
|
476
|
+
Empty list = the plan would be safe to execute as a real flash.
|
|
477
|
+
Pure; no I/O.
|
|
478
|
+
"""
|
|
479
|
+
errors: list[str] = []
|
|
480
|
+
|
|
481
|
+
if plan.image.format is None:
|
|
482
|
+
# Specific guidance when the operator dropped a tarball on
|
|
483
|
+
# PIXIE_IMAGES: those wrap the actual image inside per-file TAR
|
|
484
|
+
# headers, and pixie's flash code is single-stream-only. A
|
|
485
|
+
# generic "format not recognised" would leave operators
|
|
486
|
+
# confused; tell them what to do.
|
|
487
|
+
if images.is_tarball_extension(plan.image.display):
|
|
488
|
+
errors.append(
|
|
489
|
+
f"image is a tarball, not a single-file image: "
|
|
490
|
+
f"{plan.image.display}. Extract first "
|
|
491
|
+
f"(``tar -xf {plan.image.display}``) and drop the "
|
|
492
|
+
f"resulting .img / .qcow2 / .img.zst / .img.xz / "
|
|
493
|
+
f".img.gz / .img.bz2 onto PIXIE_IMAGES."
|
|
494
|
+
)
|
|
495
|
+
else:
|
|
496
|
+
errors.append(
|
|
497
|
+
f"image format not recognised: {plan.image.display} "
|
|
498
|
+
f"(supported: .qcow2, .img, .img.zst, .img.xz, "
|
|
499
|
+
f".img.gz, .img.bz2)"
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
if not plan.target.exists:
|
|
503
|
+
errors.append(f"target does not exist: {plan.target.path}")
|
|
504
|
+
elif not plan.target.is_block_device:
|
|
505
|
+
errors.append(f"target is not a block device: {plan.target.path}")
|
|
506
|
+
|
|
507
|
+
# A mounted partition on the target means the disk is IN USE, so
|
|
508
|
+
# refuse: you can usually unmount, but the fact it was mounted means
|
|
509
|
+
# something was using it, and silently overwriting it is the wrong
|
|
510
|
+
# answer. The flasher live env must NOT auto-mount the target
|
|
511
|
+
# (systemd.gpt_auto=0 on the kernel cmdline); if it does, that's the
|
|
512
|
+
# bug to fix, not a reason to clobber the disk.
|
|
513
|
+
if plan.target.mountpoints:
|
|
514
|
+
errors.append(
|
|
515
|
+
f"target has mounted partitions ({', '.join(plan.target.mountpoints)}); "
|
|
516
|
+
"the disk is in use. Unmount it first if you really mean to overwrite it."
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
if (
|
|
520
|
+
plan.target.size_bytes is not None
|
|
521
|
+
and plan.image.virtual_size_bytes is not None
|
|
522
|
+
and plan.image.virtual_size_bytes > plan.target.size_bytes
|
|
523
|
+
):
|
|
524
|
+
errors.append(
|
|
525
|
+
f"image ({plan.image.virtual_size_bytes} bytes) "
|
|
526
|
+
f"is larger than target ({plan.target.size_bytes} bytes)"
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
return errors
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
# ---------- Real write -------------------------------------------------------
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
class FlashError(RuntimeError):
|
|
536
|
+
"""Raised when a flash-related operation cannot complete.
|
|
537
|
+
|
|
538
|
+
:class:`FlashRaceError` is a subclass for the specific case where
|
|
539
|
+
the target's state changed between the last successful probe and
|
|
540
|
+
the attempted write (it became mounted, stopped being a block
|
|
541
|
+
device, etc.) -- ``pixie`` surfaces that as exit code 5.
|
|
542
|
+
"""
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
class FlashRaceError(FlashError):
|
|
546
|
+
"""The target changed state between probe and write (mounted, removed, ...)."""
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
class FlashCancelled(FlashError):
|
|
550
|
+
"""Raised when the operator's ``cancel`` callback returns True.
|
|
551
|
+
|
|
552
|
+
Distinct from :class:`FlashError` proper so callers (the
|
|
553
|
+
``pixie`` wizard, tests) can branch on "operator-requested abort"
|
|
554
|
+
vs "the underlying pipeline failed". Subclassing means callers
|
|
555
|
+
that catch
|
|
556
|
+
:class:`FlashError` still handle cancellation as a failure path
|
|
557
|
+
if they don't care about the distinction.
|
|
558
|
+
"""
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
class FlashIntegrityError(FlashError):
|
|
562
|
+
"""The fetched bytes did not hash to the source's declared digest.
|
|
563
|
+
|
|
564
|
+
Raised after a stream finishes when an ``oras://`` reference (or a
|
|
565
|
+
catalog entry carrying a ``sha256``) committed to a content digest
|
|
566
|
+
and the ``sha256sum`` computed on the wire disagrees. Subclassing
|
|
567
|
+
:class:`FlashError` means existing ``except FlashError`` handlers
|
|
568
|
+
treat a corrupted / tampered download as a flash failure; callers
|
|
569
|
+
that care can branch on the integrity-specific type.
|
|
570
|
+
"""
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _spawn_cancel_watchdog(
|
|
574
|
+
procs: list[subprocess.Popen[Any]],
|
|
575
|
+
cancel: CancelCheck | None,
|
|
576
|
+
*,
|
|
577
|
+
poll_interval: float = 0.25,
|
|
578
|
+
terminate_grace: float = 1.0,
|
|
579
|
+
) -> threading.Thread | None:
|
|
580
|
+
"""Spawn a daemon thread that polls ``cancel()`` and kills the
|
|
581
|
+
pipeline subprocesses on True.
|
|
582
|
+
|
|
583
|
+
The watchdog exits naturally when all ``procs`` have finished
|
|
584
|
+
(so it doesn't outlive a successful flash). On cancel: SIGTERM
|
|
585
|
+
each live proc, give them ``terminate_grace`` seconds to drain
|
|
586
|
+
cleanly, then SIGKILL anything still alive. The main pipeline
|
|
587
|
+
will then see non-zero exit codes / EOF on its pipes; the caller
|
|
588
|
+
re-checks ``cancel()`` after the pipeline returns and raises
|
|
589
|
+
:class:`FlashCancelled` rather than :class:`FlashError`.
|
|
590
|
+
"""
|
|
591
|
+
if cancel is None:
|
|
592
|
+
return None
|
|
593
|
+
|
|
594
|
+
def _watch() -> None:
|
|
595
|
+
while True:
|
|
596
|
+
if all(p.poll() is not None for p in procs):
|
|
597
|
+
return # natural completion: nothing left to kill
|
|
598
|
+
if cancel():
|
|
599
|
+
for p in procs:
|
|
600
|
+
if p.poll() is None:
|
|
601
|
+
with contextlib.suppress(ProcessLookupError):
|
|
602
|
+
p.terminate()
|
|
603
|
+
deadline = time.monotonic() + terminate_grace
|
|
604
|
+
for p in procs:
|
|
605
|
+
remaining = max(0.0, deadline - time.monotonic())
|
|
606
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
607
|
+
p.wait(timeout=remaining)
|
|
608
|
+
for p in procs:
|
|
609
|
+
if p.poll() is None:
|
|
610
|
+
with contextlib.suppress(ProcessLookupError):
|
|
611
|
+
p.kill()
|
|
612
|
+
return
|
|
613
|
+
time.sleep(poll_interval)
|
|
614
|
+
|
|
615
|
+
thread = threading.Thread(target=_watch, daemon=True)
|
|
616
|
+
thread.start()
|
|
617
|
+
return thread
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def execute_plan(
|
|
621
|
+
plan: FlashPlan,
|
|
622
|
+
*,
|
|
623
|
+
progress: ProgressCallback | None = None,
|
|
624
|
+
cancel: CancelCheck | None = None,
|
|
625
|
+
) -> None:
|
|
626
|
+
"""Write ``plan.image`` to ``plan.target``.
|
|
627
|
+
|
|
628
|
+
Re-probes the target immediately before writing to catch races
|
|
629
|
+
(target gets mounted, swapped, or removed between the dry-run and
|
|
630
|
+
the real flash). Dispatches to the right write strategy based on
|
|
631
|
+
image format. Synchronises and re-reads the partition table on
|
|
632
|
+
success.
|
|
633
|
+
|
|
634
|
+
If ``progress`` is given, lifecycle :class:`FlashProgress` events
|
|
635
|
+
are emitted: ``started``, ``writing``, ``synced``, ``partprobed``.
|
|
636
|
+
On any :class:`FlashError`, a ``failed`` event is emitted with the
|
|
637
|
+
exception string in ``note`` and the exception re-raised.
|
|
638
|
+
|
|
639
|
+
If ``cancel`` is given (a zero-arg callable returning ``bool``), a
|
|
640
|
+
watchdog thread polls it ~4Hz while a URL flash is streaming. On
|
|
641
|
+
True, the pipeline's subprocesses (``curl`` / decompressor /
|
|
642
|
+
``dd``) are SIGTERM'd with a 1s grace then SIGKILL'd; the call
|
|
643
|
+
then raises :class:`FlashCancelled` rather than
|
|
644
|
+
:class:`FlashError`. Cancel applies only to the URL flash paths
|
|
645
|
+
(where a slow remote can leave the operator waiting); the
|
|
646
|
+
local-file dispatch finishes in a few seconds and isn't worth
|
|
647
|
+
interrupting.
|
|
648
|
+
|
|
649
|
+
Raises :class:`FlashError` for caller-visible failures (target no
|
|
650
|
+
longer suitable, format unrecognised, write subprocess failed).
|
|
651
|
+
Raises :class:`FlashCancelled` when the operator's cancel
|
|
652
|
+
callback returned True.
|
|
653
|
+
"""
|
|
654
|
+
_emit(progress, "started", total_bytes=plan.image.virtual_size_bytes)
|
|
655
|
+
|
|
656
|
+
try:
|
|
657
|
+
fresh_target = probe_target(plan.target.path)
|
|
658
|
+
if not fresh_target.exists or not fresh_target.is_block_device:
|
|
659
|
+
raise FlashRaceError(f"target is no longer a block device: {plan.target.path}")
|
|
660
|
+
if fresh_target.mountpoints:
|
|
661
|
+
# A mounted partition means the disk is IN USE -- refuse.
|
|
662
|
+
# Partitions + data are fine (pixie overwrites whole disks);
|
|
663
|
+
# only a *mount* signals something is using it. We do NOT
|
|
664
|
+
# auto-unmount: you usually could, but the fact it was
|
|
665
|
+
# mounted means it was in use. The flasher live env must not
|
|
666
|
+
# auto-mount the target (systemd.gpt_auto=0); a mount here
|
|
667
|
+
# is therefore a real one, not the live env being silly.
|
|
668
|
+
raise FlashRaceError(
|
|
669
|
+
f"target has mounted partitions ({', '.join(fresh_target.mountpoints)}); "
|
|
670
|
+
"the disk is in use. Unmount it first if you really mean to overwrite it."
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
fmt = plan.image.format
|
|
674
|
+
total_bytes = plan.image.virtual_size_bytes
|
|
675
|
+
_emit(progress, "writing", note=fmt or "?")
|
|
676
|
+
if plan.image.url is not None:
|
|
677
|
+
# Streaming pipeline: curl URL | (optional zstd -d) | dd -> target.
|
|
678
|
+
# qcow2 can't stream-convert (random-access), so it's downloaded
|
|
679
|
+
# to a temp file first and then handed to the existing local
|
|
680
|
+
# qcow2 path.
|
|
681
|
+
expected_sha = plan.image.expected_sha
|
|
682
|
+
if fmt == "img":
|
|
683
|
+
_flash_img_from_url(
|
|
684
|
+
plan.image.url,
|
|
685
|
+
plan.target.path,
|
|
686
|
+
progress=progress,
|
|
687
|
+
total_bytes=total_bytes,
|
|
688
|
+
cancel=cancel,
|
|
689
|
+
expected_sha=expected_sha,
|
|
690
|
+
)
|
|
691
|
+
elif fmt == "img.zst":
|
|
692
|
+
_flash_zst_from_url(
|
|
693
|
+
plan.image.url,
|
|
694
|
+
plan.target.path,
|
|
695
|
+
progress=progress,
|
|
696
|
+
total_bytes=total_bytes,
|
|
697
|
+
cancel=cancel,
|
|
698
|
+
expected_sha=expected_sha,
|
|
699
|
+
)
|
|
700
|
+
elif fmt == "img.xz":
|
|
701
|
+
_flash_xz_from_url(
|
|
702
|
+
plan.image.url,
|
|
703
|
+
plan.target.path,
|
|
704
|
+
progress=progress,
|
|
705
|
+
total_bytes=total_bytes,
|
|
706
|
+
cancel=cancel,
|
|
707
|
+
expected_sha=expected_sha,
|
|
708
|
+
)
|
|
709
|
+
elif fmt == "img.gz":
|
|
710
|
+
_flash_gz_from_url(
|
|
711
|
+
plan.image.url,
|
|
712
|
+
plan.target.path,
|
|
713
|
+
progress=progress,
|
|
714
|
+
total_bytes=total_bytes,
|
|
715
|
+
cancel=cancel,
|
|
716
|
+
expected_sha=expected_sha,
|
|
717
|
+
)
|
|
718
|
+
elif fmt == "img.bz2":
|
|
719
|
+
_flash_bz2_from_url(
|
|
720
|
+
plan.image.url,
|
|
721
|
+
plan.target.path,
|
|
722
|
+
progress=progress,
|
|
723
|
+
total_bytes=total_bytes,
|
|
724
|
+
cancel=cancel,
|
|
725
|
+
expected_sha=expected_sha,
|
|
726
|
+
)
|
|
727
|
+
elif fmt == "qcow2":
|
|
728
|
+
_flash_qcow2_from_url(
|
|
729
|
+
plan.image.url,
|
|
730
|
+
plan.target.path,
|
|
731
|
+
cancel=cancel,
|
|
732
|
+
expected_sha=expected_sha,
|
|
733
|
+
)
|
|
734
|
+
else:
|
|
735
|
+
raise FlashError(f"cannot flash image of format {fmt!r}")
|
|
736
|
+
else:
|
|
737
|
+
assert plan.image.path is not None # typer narrows; validate_plan guarantees
|
|
738
|
+
if fmt == "img":
|
|
739
|
+
_flash_img(
|
|
740
|
+
plan.image.path,
|
|
741
|
+
plan.target.path,
|
|
742
|
+
progress=progress,
|
|
743
|
+
total_bytes=total_bytes,
|
|
744
|
+
)
|
|
745
|
+
elif fmt == "img.zst":
|
|
746
|
+
_flash_zst(
|
|
747
|
+
plan.image.path,
|
|
748
|
+
plan.target.path,
|
|
749
|
+
progress=progress,
|
|
750
|
+
total_bytes=total_bytes,
|
|
751
|
+
)
|
|
752
|
+
elif fmt == "img.xz":
|
|
753
|
+
_flash_xz(
|
|
754
|
+
plan.image.path,
|
|
755
|
+
plan.target.path,
|
|
756
|
+
progress=progress,
|
|
757
|
+
total_bytes=total_bytes,
|
|
758
|
+
)
|
|
759
|
+
elif fmt == "img.gz":
|
|
760
|
+
_flash_gz(
|
|
761
|
+
plan.image.path,
|
|
762
|
+
plan.target.path,
|
|
763
|
+
progress=progress,
|
|
764
|
+
total_bytes=total_bytes,
|
|
765
|
+
)
|
|
766
|
+
elif fmt == "img.bz2":
|
|
767
|
+
_flash_bz2(
|
|
768
|
+
plan.image.path,
|
|
769
|
+
plan.target.path,
|
|
770
|
+
progress=progress,
|
|
771
|
+
total_bytes=total_bytes,
|
|
772
|
+
)
|
|
773
|
+
elif fmt == "qcow2":
|
|
774
|
+
_flash_qcow2(plan.image.path, plan.target.path)
|
|
775
|
+
else:
|
|
776
|
+
raise FlashError(f"cannot flash image of format {fmt!r}")
|
|
777
|
+
|
|
778
|
+
_sync_target(plan.target.path)
|
|
779
|
+
_emit(progress, "synced")
|
|
780
|
+
|
|
781
|
+
_partprobe_target(plan.target.path)
|
|
782
|
+
_emit(progress, "partprobed")
|
|
783
|
+
except FlashError as exc:
|
|
784
|
+
_emit(progress, "failed", note=str(exc))
|
|
785
|
+
raise
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
# GPT partition-type GUID of an EFI System Partition (lowercase, as
|
|
789
|
+
# ``lsblk`` reports it).
|
|
790
|
+
_ESP_TYPE_GUID = "c12a7328-f81f-11d2-ba4b-00a0c93ec93b"
|
|
791
|
+
# The UEFI removable-media fallback loader path. dd'd images almost
|
|
792
|
+
# always carry this (it's what lets a stick / disk boot without an
|
|
793
|
+
# NVRAM entry); backslash-separated because efibootmgr wants an EFI
|
|
794
|
+
# path, not a POSIX one.
|
|
795
|
+
_UEFI_FALLBACK_LOADER = "\\EFI\\BOOT\\BOOTX64.EFI"
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _efibootmgr(args: list[str] | None = None) -> str:
|
|
799
|
+
"""Run ``efibootmgr`` and return stdout. Raises on non-zero exit."""
|
|
800
|
+
return subprocess.run(
|
|
801
|
+
["efibootmgr", *(args or [])],
|
|
802
|
+
check=True,
|
|
803
|
+
capture_output=True,
|
|
804
|
+
text=True,
|
|
805
|
+
timeout=30,
|
|
806
|
+
).stdout
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _boot_entries_with_label(efibootmgr_out: str, label: str) -> list[str]:
|
|
810
|
+
"""Entry numbers (``Boot####``) whose description equals ``label``.
|
|
811
|
+
|
|
812
|
+
efibootmgr prints ``Boot0007* <label>\\t<device-path>``; we match
|
|
813
|
+
the label portion so reflashes can drop their own prior entries
|
|
814
|
+
without touching the operator's / firmware's entries.
|
|
815
|
+
"""
|
|
816
|
+
nums: list[str] = []
|
|
817
|
+
for line in efibootmgr_out.splitlines():
|
|
818
|
+
m = re.match(r"^Boot([0-9A-Fa-f]{4})\*?\s+(.*)$", line)
|
|
819
|
+
if m and m.group(2).split("\t", 1)[0].strip() == label:
|
|
820
|
+
nums.append(m.group(1))
|
|
821
|
+
return nums
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _find_esp_partition_number(disk: Path) -> int | None:
|
|
825
|
+
"""Return the 1-based partition number of ``disk``'s EFI System
|
|
826
|
+
Partition, or ``None`` if it has none. Reads ``lsblk -J``."""
|
|
827
|
+
try:
|
|
828
|
+
out = subprocess.run(
|
|
829
|
+
["lsblk", "-J", "-o", "PATH,PARTTYPE,PARTN", str(disk)],
|
|
830
|
+
check=True,
|
|
831
|
+
capture_output=True,
|
|
832
|
+
text=True,
|
|
833
|
+
timeout=15,
|
|
834
|
+
).stdout
|
|
835
|
+
data = json.loads(out)
|
|
836
|
+
except (FileNotFoundError, subprocess.SubprocessError, OSError, ValueError):
|
|
837
|
+
return None
|
|
838
|
+
for dev in data.get("blockdevices", []):
|
|
839
|
+
for child in dev.get("children") or []:
|
|
840
|
+
if (child.get("parttype") or "").lower() != _ESP_TYPE_GUID:
|
|
841
|
+
continue
|
|
842
|
+
partn = child.get("partn")
|
|
843
|
+
if partn is not None:
|
|
844
|
+
try:
|
|
845
|
+
return int(partn)
|
|
846
|
+
except (TypeError, ValueError):
|
|
847
|
+
pass
|
|
848
|
+
# Older lsblk has no PARTN: fall back to the trailing digits
|
|
849
|
+
# of the device path (/dev/sda1 -> 1, /dev/nvme0n1p1 -> 1).
|
|
850
|
+
m = re.search(r"(\d+)$", child.get("path") or "")
|
|
851
|
+
if m:
|
|
852
|
+
return int(m.group(1))
|
|
853
|
+
return None
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def register_uefi_boot_entry(target_disk: Path, *, label: str = "pixie flashed") -> str:
|
|
857
|
+
"""Register (and select for next boot) a UEFI NVRAM entry for a
|
|
858
|
+
freshly-flashed disk, returning a one-line human status.
|
|
859
|
+
|
|
860
|
+
A ``dd``-written image carries an EFI System Partition with a
|
|
861
|
+
bootloader but NO firmware boot entry (NVRAM lives in the firmware,
|
|
862
|
+
not the image), so a UEFI box has nothing in its boot order to boot
|
|
863
|
+
and falls back to netboot -- the box never comes up on its new OS.
|
|
864
|
+
This points the firmware at the ESP's fallback loader:
|
|
865
|
+
|
|
866
|
+
* creates an entry for ``\\EFI\\BOOT\\BOOTX64.EFI`` on the ESP,
|
|
867
|
+
* keeps the existing entries (the netboot entry the box just used)
|
|
868
|
+
FIRST in BootOrder so pixie stays in control on later boots, with
|
|
869
|
+
the new entry appended,
|
|
870
|
+
* sets BootNext to the new entry so THIS reboot lands on the disk
|
|
871
|
+
regardless of whether the firmware falls through on iPXE exit.
|
|
872
|
+
|
|
873
|
+
Best-effort and UEFI-only: returns a "skipped ..." status (rather
|
|
874
|
+
than raising) when the box isn't UEFI, ``efibootmgr`` is absent, or
|
|
875
|
+
the disk has no ESP. A genuine ``efibootmgr`` failure propagates as
|
|
876
|
+
``CalledProcessError`` for the caller to log.
|
|
877
|
+
"""
|
|
878
|
+
if not Path("/sys/firmware/efi/efivars").is_dir():
|
|
879
|
+
return "skipped UEFI boot entry: box is not booted in UEFI mode (BIOS/CSM)"
|
|
880
|
+
if shutil.which("efibootmgr") is None:
|
|
881
|
+
return "skipped UEFI boot entry: efibootmgr not installed in the live env"
|
|
882
|
+
part = _find_esp_partition_number(target_disk)
|
|
883
|
+
if part is None:
|
|
884
|
+
return f"skipped UEFI boot entry: no EFI System Partition found on {target_disk}"
|
|
885
|
+
|
|
886
|
+
# Idempotent across reflashes: drop our own prior entries first.
|
|
887
|
+
for num in _boot_entries_with_label(_efibootmgr(), label):
|
|
888
|
+
_efibootmgr(["-b", num, "-B"])
|
|
889
|
+
|
|
890
|
+
# ``--create-only`` registers the entry WITHOUT adding it to
|
|
891
|
+
# BootOrder. We deliberately NEVER rewrite BootOrder: doing so on
|
|
892
|
+
# server firmware risks dropping the box's real boot entries -- an
|
|
893
|
+
# earlier version that did ``-o <old_order>,<new>`` stranded an EPYC
|
|
894
|
+
# box out of UEFI when the old-order parse came back short. Instead
|
|
895
|
+
# we set only the one-shot ``BootNext``: the firmware consumes it
|
|
896
|
+
# after a single boot and the standing boot order is left untouched.
|
|
897
|
+
_efibootmgr(
|
|
898
|
+
[
|
|
899
|
+
"--create-only",
|
|
900
|
+
"--disk",
|
|
901
|
+
str(target_disk),
|
|
902
|
+
"--part",
|
|
903
|
+
str(part),
|
|
904
|
+
"--loader",
|
|
905
|
+
_UEFI_FALLBACK_LOADER,
|
|
906
|
+
"--label",
|
|
907
|
+
label,
|
|
908
|
+
]
|
|
909
|
+
)
|
|
910
|
+
# We just deleted any prior pixie entries, so the entry carrying our
|
|
911
|
+
# label now is the one we just created.
|
|
912
|
+
nums = _boot_entries_with_label(_efibootmgr(), label)
|
|
913
|
+
if not nums:
|
|
914
|
+
return f"created UEFI boot entry on {target_disk} (could not confirm BootNext)"
|
|
915
|
+
new = nums[-1]
|
|
916
|
+
_efibootmgr(["-n", new]) # one-shot BootNext; BootOrder untouched
|
|
917
|
+
return (
|
|
918
|
+
f"registered UEFI boot entry Boot{new} -> {target_disk} "
|
|
919
|
+
f"(ESP partition {part}); BootNext set for the next boot, BootOrder untouched"
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
# Defensive scrub for subprocess stderr surfaced to the operator: an
|
|
924
|
+
# ``oras://`` flash injects a short-lived bearer via ``-H Authorization:
|
|
925
|
+
# Bearer <token>``, and curl could echo that header (e.g. on a verbose
|
|
926
|
+
# build or some error paths). Redact the token before it reaches the
|
|
927
|
+
# progress UI / logs so a captured stream can't replay the credential.
|
|
928
|
+
_BEARER_RE = re.compile(r"(?i)(authorization:\s*bearer\s+|bearer\s+)[A-Za-z0-9._~+/=-]+")
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _redact_secrets(line: str) -> str:
|
|
932
|
+
"""Replace any ``Bearer <token>`` in a log line with a placeholder."""
|
|
933
|
+
return _BEARER_RE.sub(r"\1<redacted>", line)
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _start_subprocess_log_pump(
|
|
937
|
+
proc: subprocess.Popen[Any],
|
|
938
|
+
progress: ProgressCallback | None,
|
|
939
|
+
label: str,
|
|
940
|
+
) -> threading.Thread | None:
|
|
941
|
+
"""Drain ``proc.stderr`` line-by-line and emit ``subprocess_log``
|
|
942
|
+
events to the progress callback.
|
|
943
|
+
|
|
944
|
+
Used for auxiliary pipeline processes (zstd / gzip / xz / bzip2 /
|
|
945
|
+
curl) when a progress callback is set (the ``pixie`` wizard). The
|
|
946
|
+
wizard prints each line via ``console.print`` inside its ``with
|
|
947
|
+
Progress():`` context; Rich routes the line above the progress
|
|
948
|
+
widget without corrupting it.
|
|
949
|
+
|
|
950
|
+
Lines are decoded as UTF-8 with replacement. The reader is
|
|
951
|
+
newline-bound, so subprocesses that update via carriage-return-
|
|
952
|
+
only refresh (curl's progress bar, zstd's --no-progress=auto)
|
|
953
|
+
don't emit until they finally write a ``\\n`` -- exactly what we
|
|
954
|
+
want, since those refresh streams would otherwise spam the
|
|
955
|
+
progress widget.
|
|
956
|
+
|
|
957
|
+
Returns the thread (caller ``.join()``s after the proc exits) or
|
|
958
|
+
``None`` if no callback is set.
|
|
959
|
+
"""
|
|
960
|
+
if progress is None or proc.stderr is None:
|
|
961
|
+
return None
|
|
962
|
+
|
|
963
|
+
def _pump() -> None:
|
|
964
|
+
stream = proc.stderr
|
|
965
|
+
if stream is None:
|
|
966
|
+
return
|
|
967
|
+
for raw in stream:
|
|
968
|
+
line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
969
|
+
line = line.rstrip("\r\n")
|
|
970
|
+
if not line:
|
|
971
|
+
continue
|
|
972
|
+
_emit(progress, "subprocess_log", note=f"{label}: {_redact_secrets(line)}")
|
|
973
|
+
|
|
974
|
+
thread = threading.Thread(target=_pump, daemon=True, name=f"pixie-{label}-stderr")
|
|
975
|
+
thread.start()
|
|
976
|
+
return thread
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
def _start_dd_progress_thread(
|
|
980
|
+
proc: subprocess.Popen[str],
|
|
981
|
+
progress: ProgressCallback | None,
|
|
982
|
+
total_bytes: int | None,
|
|
983
|
+
*,
|
|
984
|
+
event: str = "writing_progress",
|
|
985
|
+
bytes_field: str = "bytes_written",
|
|
986
|
+
) -> threading.Thread | None:
|
|
987
|
+
"""Spawn the dd-stderr pump if a progress callback is provided.
|
|
988
|
+
|
|
989
|
+
Returns the thread (so the caller can ``.join()`` after dd exits)
|
|
990
|
+
or ``None`` if no callback was given. When ``progress`` is ``None``
|
|
991
|
+
the caller leaves dd's stderr inherited and dd's status=progress
|
|
992
|
+
output goes to the operator's terminal as before.
|
|
993
|
+
|
|
994
|
+
``event`` + ``bytes_field`` are forwarded to :func:`_pump_dd_progress`
|
|
995
|
+
so the same helper can drive either the disk-write progress bar
|
|
996
|
+
(defaults) or the network-side download progress bar
|
|
997
|
+
(``"downloading_progress"`` / ``"bytes_downloaded"``).
|
|
998
|
+
"""
|
|
999
|
+
if progress is None or proc.stderr is None:
|
|
1000
|
+
return None
|
|
1001
|
+
thread = threading.Thread(
|
|
1002
|
+
target=_pump_dd_progress,
|
|
1003
|
+
args=(proc.stderr, progress, total_bytes),
|
|
1004
|
+
kwargs={"event": event, "bytes_field": bytes_field},
|
|
1005
|
+
daemon=True,
|
|
1006
|
+
)
|
|
1007
|
+
thread.start()
|
|
1008
|
+
return thread
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def _spawn_download_meter(
|
|
1012
|
+
upstream_stdout: IO[bytes],
|
|
1013
|
+
progress: ProgressCallback | None,
|
|
1014
|
+
total_bytes: int | None,
|
|
1015
|
+
) -> tuple[subprocess.Popen[Any] | None, threading.Thread | None]:
|
|
1016
|
+
"""Insert a ``dd bs=1M status=progress`` between curl and the next
|
|
1017
|
+
pipeline stage so the operator gets a download progress bar
|
|
1018
|
+
distinct from the disk-write one.
|
|
1019
|
+
|
|
1020
|
+
The interposed dd just shuffles bytes through: it reads curl's
|
|
1021
|
+
output in 1 MiB chunks and writes them to its own stdout
|
|
1022
|
+
(which the caller then feeds into the next stage's stdin). The
|
|
1023
|
+
progress meter on its stderr is what we mine for the
|
|
1024
|
+
``downloading_progress`` events; total throughput is unaffected
|
|
1025
|
+
in practice -- the pipe is byte-flow-bound by network speed,
|
|
1026
|
+
not by dd's passthrough.
|
|
1027
|
+
|
|
1028
|
+
Returns ``(proc, pump_thread)``. When ``progress`` is None the
|
|
1029
|
+
caller does not want a download bar; we return ``(None, None)``
|
|
1030
|
+
and the caller uses ``upstream_stdout`` directly.
|
|
1031
|
+
"""
|
|
1032
|
+
if progress is None:
|
|
1033
|
+
return None, None
|
|
1034
|
+
proc = subprocess.Popen(
|
|
1035
|
+
# ``iflag=fullblock`` makes dd accumulate a full 1 MiB before
|
|
1036
|
+
# writing, avoiding short-read chunks that would partial-count
|
|
1037
|
+
# against the progress meter on a slow pipe.
|
|
1038
|
+
["dd", "bs=1M", "status=progress", "iflag=fullblock"],
|
|
1039
|
+
stdin=upstream_stdout,
|
|
1040
|
+
stdout=subprocess.PIPE,
|
|
1041
|
+
stderr=subprocess.PIPE,
|
|
1042
|
+
text=True,
|
|
1043
|
+
)
|
|
1044
|
+
pump = _start_dd_progress_thread(
|
|
1045
|
+
proc,
|
|
1046
|
+
progress,
|
|
1047
|
+
total_bytes,
|
|
1048
|
+
event="downloading_progress",
|
|
1049
|
+
bytes_field="bytes_downloaded",
|
|
1050
|
+
)
|
|
1051
|
+
return proc, pump
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def _flash_img(
|
|
1055
|
+
image: Path,
|
|
1056
|
+
target: Path,
|
|
1057
|
+
*,
|
|
1058
|
+
progress: ProgressCallback | None = None,
|
|
1059
|
+
total_bytes: int | None = None,
|
|
1060
|
+
) -> None:
|
|
1061
|
+
"""Write a raw .img to a block device with ``dd``.
|
|
1062
|
+
|
|
1063
|
+
``oflag=direct`` + ``conv=fsync`` are both required: O_DIRECT
|
|
1064
|
+
bypasses the kernel page cache so the running OS's binaries
|
|
1065
|
+
aren't shadowed by stale-pre-flash pages if the target happens
|
|
1066
|
+
to be the disk we booted from, and conv=fsync makes dd return
|
|
1067
|
+
only when its writes have hit disk. Combining the two is the
|
|
1068
|
+
only way an in-place reflash leaves a consistent on-disk state.
|
|
1069
|
+
"""
|
|
1070
|
+
cmd = [
|
|
1071
|
+
"dd",
|
|
1072
|
+
f"if={image}",
|
|
1073
|
+
f"of={target}",
|
|
1074
|
+
"bs=4M",
|
|
1075
|
+
"oflag=direct",
|
|
1076
|
+
"conv=fsync",
|
|
1077
|
+
"status=progress",
|
|
1078
|
+
]
|
|
1079
|
+
stderr = subprocess.PIPE if progress is not None else None
|
|
1080
|
+
proc = subprocess.Popen(cmd, stderr=stderr, text=True)
|
|
1081
|
+
pump = _start_dd_progress_thread(proc, progress, total_bytes)
|
|
1082
|
+
rc = proc.wait()
|
|
1083
|
+
if pump is not None:
|
|
1084
|
+
pump.join(timeout=2)
|
|
1085
|
+
if rc != 0:
|
|
1086
|
+
raise FlashError(f"dd exited {rc} writing {image} -> {target}")
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _flash_compressed(
|
|
1090
|
+
image: Path,
|
|
1091
|
+
target: Path,
|
|
1092
|
+
decompress_cmd: list[str],
|
|
1093
|
+
decompress_name: str,
|
|
1094
|
+
*,
|
|
1095
|
+
progress: ProgressCallback | None = None,
|
|
1096
|
+
total_bytes: int | None = None,
|
|
1097
|
+
) -> None:
|
|
1098
|
+
"""Pipeline ``<decompress_cmd> | dd of=TARGET ...``.
|
|
1099
|
+
|
|
1100
|
+
Generic single-file-decompressor + dd pipeline used by every
|
|
1101
|
+
``.img.<algo>`` writer. ``decompress_cmd`` reads the image
|
|
1102
|
+
(typically as a positional arg or via ``--stdout``-style flag)
|
|
1103
|
+
and writes raw decompressed bytes to its stdout, which dd
|
|
1104
|
+
consumes. ``decompress_name`` is used in error messages.
|
|
1105
|
+
|
|
1106
|
+
NOTE: this only handles SINGLE-FILE compression streams (zstd,
|
|
1107
|
+
xz, gzip, bzip2). It does NOT handle ``.tar.gz`` /
|
|
1108
|
+
``.tar.xz`` / ``.zip`` containers -- those wrap one-or-many
|
|
1109
|
+
files inside metadata, and dd'ing a decompressed tar stream
|
|
1110
|
+
would write tar headers into the target's MBR. Format
|
|
1111
|
+
detection in ``images.py`` deliberately rejects tarball
|
|
1112
|
+
extensions.
|
|
1113
|
+
"""
|
|
1114
|
+
# When a progress callback is set (``pixie`` wizard caller), pipe
|
|
1115
|
+
# the decompressor's stderr into a pump thread that emits
|
|
1116
|
+
# ``subprocess_log`` events; the wizard routes those through
|
|
1117
|
+
# Rich's console so they print above the progress widget without
|
|
1118
|
+
# corrupting it. Callers without a progress callback leave stderr
|
|
1119
|
+
# inherited (operator's tty sees zstd/gzip output natively).
|
|
1120
|
+
decomp_stderr = subprocess.PIPE if progress is not None else None
|
|
1121
|
+
decomp_proc = subprocess.Popen(decompress_cmd, stdout=subprocess.PIPE, stderr=decomp_stderr)
|
|
1122
|
+
decomp_log_pump = _start_subprocess_log_pump(decomp_proc, progress, decompress_name)
|
|
1123
|
+
try:
|
|
1124
|
+
stderr = subprocess.PIPE if progress is not None else None
|
|
1125
|
+
dd_proc = subprocess.Popen(
|
|
1126
|
+
[
|
|
1127
|
+
"dd",
|
|
1128
|
+
f"of={target}",
|
|
1129
|
+
"bs=4M",
|
|
1130
|
+
"oflag=direct",
|
|
1131
|
+
"conv=fsync",
|
|
1132
|
+
"status=progress",
|
|
1133
|
+
],
|
|
1134
|
+
stdin=decomp_proc.stdout,
|
|
1135
|
+
stderr=stderr,
|
|
1136
|
+
text=True,
|
|
1137
|
+
)
|
|
1138
|
+
pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
|
|
1139
|
+
# Let the decompressor see SIGPIPE if dd exits early.
|
|
1140
|
+
if decomp_proc.stdout is not None:
|
|
1141
|
+
decomp_proc.stdout.close()
|
|
1142
|
+
dd_rc = dd_proc.wait()
|
|
1143
|
+
if pump is not None:
|
|
1144
|
+
pump.join(timeout=2)
|
|
1145
|
+
finally:
|
|
1146
|
+
decomp_rc = decomp_proc.wait()
|
|
1147
|
+
if decomp_log_pump is not None:
|
|
1148
|
+
decomp_log_pump.join(timeout=2)
|
|
1149
|
+
|
|
1150
|
+
if dd_rc != 0:
|
|
1151
|
+
raise FlashError(f"dd exited {dd_rc} writing {image} -> {target}")
|
|
1152
|
+
if decomp_rc != 0:
|
|
1153
|
+
raise FlashError(f"{decompress_name} exited {decomp_rc} decompressing {image}")
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
def _flash_zst(
|
|
1157
|
+
image: Path,
|
|
1158
|
+
target: Path,
|
|
1159
|
+
*,
|
|
1160
|
+
progress: ProgressCallback | None = None,
|
|
1161
|
+
total_bytes: int | None = None,
|
|
1162
|
+
) -> None:
|
|
1163
|
+
"""Pipeline ``zstd -d --stdout IMG | dd of=TARGET ...``."""
|
|
1164
|
+
_flash_compressed(
|
|
1165
|
+
image,
|
|
1166
|
+
target,
|
|
1167
|
+
["zstd", "-d", "--stdout", str(image)],
|
|
1168
|
+
"zstd",
|
|
1169
|
+
progress=progress,
|
|
1170
|
+
total_bytes=total_bytes,
|
|
1171
|
+
)
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _flash_xz(
|
|
1175
|
+
image: Path,
|
|
1176
|
+
target: Path,
|
|
1177
|
+
*,
|
|
1178
|
+
progress: ProgressCallback | None = None,
|
|
1179
|
+
total_bytes: int | None = None,
|
|
1180
|
+
) -> None:
|
|
1181
|
+
"""Pipeline ``xz -d --stdout IMG | dd of=TARGET ...``.
|
|
1182
|
+
|
|
1183
|
+
xz decompresses at ~50-100 MB/s vs zstd's ~800-1500 MB/s;
|
|
1184
|
+
pixie's own target images ship as .img.zst for the per-job
|
|
1185
|
+
CI reflash hot path, but this writer accepts operator-supplied
|
|
1186
|
+
.img.xz so neither format is forced on operators.
|
|
1187
|
+
"""
|
|
1188
|
+
_flash_compressed(
|
|
1189
|
+
image,
|
|
1190
|
+
target,
|
|
1191
|
+
["xz", "-d", "--stdout", str(image)],
|
|
1192
|
+
"xz",
|
|
1193
|
+
progress=progress,
|
|
1194
|
+
total_bytes=total_bytes,
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def _flash_gz(
|
|
1199
|
+
image: Path,
|
|
1200
|
+
target: Path,
|
|
1201
|
+
*,
|
|
1202
|
+
progress: ProgressCallback | None = None,
|
|
1203
|
+
total_bytes: int | None = None,
|
|
1204
|
+
) -> None:
|
|
1205
|
+
"""Pipeline ``gzip -d --stdout IMG | dd of=TARGET ...``.
|
|
1206
|
+
|
|
1207
|
+
gzip is universally available and many older distro images
|
|
1208
|
+
still ship as .img.gz (Raspberry Pi OS pre-2022, older
|
|
1209
|
+
Ubuntu Server cloud images, vendor appliance bundles).
|
|
1210
|
+
Decompression is fast (~300-500 MB/s) but compression ratio
|
|
1211
|
+
is weaker than xz/zstd on zero-heavy images.
|
|
1212
|
+
"""
|
|
1213
|
+
_flash_compressed(
|
|
1214
|
+
image,
|
|
1215
|
+
target,
|
|
1216
|
+
["gzip", "-d", "--stdout", str(image)],
|
|
1217
|
+
"gzip",
|
|
1218
|
+
progress=progress,
|
|
1219
|
+
total_bytes=total_bytes,
|
|
1220
|
+
)
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _flash_bz2(
|
|
1224
|
+
image: Path,
|
|
1225
|
+
target: Path,
|
|
1226
|
+
*,
|
|
1227
|
+
progress: ProgressCallback | None = None,
|
|
1228
|
+
total_bytes: int | None = None,
|
|
1229
|
+
) -> None:
|
|
1230
|
+
"""Pipeline ``bzip2 -d --stdout IMG | dd of=TARGET ...``.
|
|
1231
|
+
|
|
1232
|
+
Decompression is the slowest of the supported formats
|
|
1233
|
+
(~10-30 MB/s) and bz2 lacks a metadata header for uncompressed
|
|
1234
|
+
size, so ``virtual_size_bytes`` is always ``None`` for .img.bz2 and
|
|
1235
|
+
validate_plan skips the size-fits-target check with a note.
|
|
1236
|
+
"""
|
|
1237
|
+
_flash_compressed(
|
|
1238
|
+
image,
|
|
1239
|
+
target,
|
|
1240
|
+
["bzip2", "-d", "--stdout", str(image)],
|
|
1241
|
+
"bzip2",
|
|
1242
|
+
progress=progress,
|
|
1243
|
+
total_bytes=total_bytes,
|
|
1244
|
+
)
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def _flash_qcow2(image: Path, target: Path) -> None:
|
|
1248
|
+
"""Write a qcow2 to a block device by converting to raw in place.
|
|
1249
|
+
|
|
1250
|
+
Byte-level progress for qcow2 isn't plumbed through to the
|
|
1251
|
+
``writing_progress`` event, so ``-p`` is dropped and stderr is
|
|
1252
|
+
captured instead: a failed convert then surfaces qemu-img's actual
|
|
1253
|
+
diagnostic (``Could not open ...``, permission denied, a corrupt
|
|
1254
|
+
-image message) in the ``FlashError`` rather than a bare exit code,
|
|
1255
|
+
which is what an operator needs when a block-device write fails.
|
|
1256
|
+
"""
|
|
1257
|
+
cmd = ["qemu-img", "convert", "-O", "raw", str(image), str(target)]
|
|
1258
|
+
proc = subprocess.run(cmd, check=False, stderr=subprocess.PIPE, text=True)
|
|
1259
|
+
if proc.returncode != 0:
|
|
1260
|
+
detail = (proc.stderr or "").strip() or "no stderr captured"
|
|
1261
|
+
raise FlashError(
|
|
1262
|
+
f"qemu-img convert exited {proc.returncode} writing {image} -> {target}: {detail}"
|
|
1263
|
+
)
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
# ---------- URL-streaming variants -------------------------------------------
|
|
1267
|
+
#
|
|
1268
|
+
# curl is the HTTP downloader: available on every Debian/Ubuntu/macOS
|
|
1269
|
+
# host the project supports, and well-instrumented for progress
|
|
1270
|
+
# reporting via ``--progress-bar`` to stderr. The pipelines mirror
|
|
1271
|
+
# the local-file flash functions but with curl on the front instead
|
|
1272
|
+
# of an open(file).
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
# ``-fsSL``:
|
|
1276
|
+
# -f: fail on HTTP errors (4xx/5xx exit non-zero)
|
|
1277
|
+
# -s: silent (no progress meter, no diagnostic notes)
|
|
1278
|
+
# -S: but still show errors (without this, -s would also silence them)
|
|
1279
|
+
# -L: follow redirects
|
|
1280
|
+
# The ``-s`` is deliberate: curl's progress meter is carriage-return-
|
|
1281
|
+
# updated, which ``pixie``'s newline-bound subprocess-log pump can
|
|
1282
|
+
# only capture as the *initial* zero-state line (followed by silence
|
|
1283
|
+
# as the same line gets overwritten in place). Operators saw "all 0"
|
|
1284
|
+
# rows above the Rich progress bar; ``-s`` silences that, ``-S``
|
|
1285
|
+
# keeps real error lines flowing through.
|
|
1286
|
+
#
|
|
1287
|
+
# ``--http1.1``: force HTTP/1.1 on every streaming fetch.
|
|
1288
|
+
#
|
|
1289
|
+
# GHCR's blob CDN (pkg-containers.githubusercontent.com) and other OCI
|
|
1290
|
+
# registries fronted by HTTP/2-capable CDNs will RST_STREAM a
|
|
1291
|
+
# long-running blob transfer once the pre-signed redirect URL's TTL
|
|
1292
|
+
# expires, surfacing here as ``curl exited 92`` (CURLE_HTTP2_STREAM)
|
|
1293
|
+
# after a roughly fixed number of minutes regardless of bytes
|
|
1294
|
+
# transferred. Operators on pixie-usbboot reported this with
|
|
1295
|
+
# multi-GiB ``oras://ghcr.io/...`` images that aborted at the same
|
|
1296
|
+
# point every retry. HTTP/1.1 transfers are not subject to that
|
|
1297
|
+
# framing-layer reset, and HTTP/2 multiplexing buys us nothing for a
|
|
1298
|
+
# single large stream-to-dd transfer, so the cost is zero.
|
|
1299
|
+
#
|
|
1300
|
+
# NO ``--retry``: every curl invocation here streams into a running
|
|
1301
|
+
# ``dd`` pipeline. If curl retries on a transient network failure,
|
|
1302
|
+
# it re-fetches from byte 0; those bytes get written to disk a
|
|
1303
|
+
# SECOND time, corrupting whatever was already there. Symptom
|
|
1304
|
+
# observed on a Supermicro BMC flash: the Rich progress bar
|
|
1305
|
+
# repeatedly hit 100% then "reset" as dd kept writing past the
|
|
1306
|
+
# image's compressed-size total. For streaming-to-dd the right
|
|
1307
|
+
# behaviour is fail-fast -- the operator gets a clean error and
|
|
1308
|
+
# can re-flash from scratch instead of seeing a silently-corrupted
|
|
1309
|
+
# target. ``--retry`` would only make sense if we also passed
|
|
1310
|
+
# ``--continue-at`` and made dd resumable, which is a much bigger
|
|
1311
|
+
# refactor for a much rarer win.
|
|
1312
|
+
_CURL_BASE = ("curl", "-fsSL", "--http1.1")
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
def _curl_args_for_source(url: str) -> tuple[list[str], int | None, str | None]:
|
|
1316
|
+
"""Build curl arguments for a fetch source.
|
|
1317
|
+
|
|
1318
|
+
Plain http(s) URLs pass through unchanged. ``oras://`` references
|
|
1319
|
+
go through :mod:`withcache.oras` to resolve the manifest layer, and the
|
|
1320
|
+
resulting bearer token is injected as a ``-H Authorization``
|
|
1321
|
+
header on the curl call. Returns ``(argv, expected_size_or_None,
|
|
1322
|
+
expected_digest_or_None)`` -- the size is the manifest's declared
|
|
1323
|
+
layer size when known (a fallback ``total_bytes`` for callers that
|
|
1324
|
+
skipped HEAD), and the digest (``sha256:<hex>``) is the layer's
|
|
1325
|
+
content address, frozen at resolve time, for the streaming
|
|
1326
|
+
integrity check. Plain URLs carry neither, so both come back
|
|
1327
|
+
``None`` and the caller keeps its zero-copy path.
|
|
1328
|
+
"""
|
|
1329
|
+
if not oras.is_oras_url(url):
|
|
1330
|
+
return [*_CURL_BASE, url], None, None
|
|
1331
|
+
resolved = oras.resolve_ref(url)
|
|
1332
|
+
args = [*_CURL_BASE]
|
|
1333
|
+
for header_name, header_value in resolved.headers.items():
|
|
1334
|
+
args.extend(["-H", f"{header_name}: {header_value}"])
|
|
1335
|
+
args.append(resolved.blob_url)
|
|
1336
|
+
return args, resolved.size, resolved.digest
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
def _spawn_hash_tee(src: IO[bytes]) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]:
|
|
1340
|
+
"""Splice ``tee | sha256sum`` onto ``src`` (curl's read pipe).
|
|
1341
|
+
|
|
1342
|
+
Returns ``(tee_proc, sha_proc)``. ``tee_proc.stdout`` carries the
|
|
1343
|
+
bytes onward to the next stage (dd / decompressor); ``sha_proc``
|
|
1344
|
+
hashes the duplicated copy and emits ``<hex> -`` once the stream
|
|
1345
|
+
drains. The hash runs entirely in subprocesses -- payload bytes
|
|
1346
|
+
never pass through the Python process. ``tee`` writes to
|
|
1347
|
+
``/dev/fd/N`` (N = the sha pipe's write end, kept open across the
|
|
1348
|
+
fork via ``pass_fds``); the parent's copy of that write end is
|
|
1349
|
+
dropped here so ``sha256sum`` sees EOF when ``tee`` exits.
|
|
1350
|
+
"""
|
|
1351
|
+
sha_proc = subprocess.Popen(["sha256sum"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
1352
|
+
assert sha_proc.stdin is not None
|
|
1353
|
+
sha_fd = sha_proc.stdin.fileno()
|
|
1354
|
+
tee_proc = subprocess.Popen(
|
|
1355
|
+
["tee", f"/dev/fd/{sha_fd}"],
|
|
1356
|
+
stdin=src,
|
|
1357
|
+
stdout=subprocess.PIPE,
|
|
1358
|
+
pass_fds=(sha_fd,),
|
|
1359
|
+
)
|
|
1360
|
+
sha_proc.stdin.close()
|
|
1361
|
+
return tee_proc, sha_proc
|
|
1362
|
+
|
|
1363
|
+
|
|
1364
|
+
def _read_observed_digest(sha_proc: subprocess.Popen[bytes]) -> str:
|
|
1365
|
+
"""Read the ``sha256:<hex>`` digest from a finished ``sha256sum``.
|
|
1366
|
+
|
|
1367
|
+
``sha256sum`` emits ``<hex> -`` once; its output is ~65 bytes so
|
|
1368
|
+
it fits the pipe buffer and never blocks the upstream pipeline.
|
|
1369
|
+
|
|
1370
|
+
Reads ``stdout`` directly rather than via ``communicate()``:
|
|
1371
|
+
:func:`_spawn_hash_tee` already closed the parent's copy of
|
|
1372
|
+
``stdin`` (to hand sha256sum its EOF), and ``communicate()`` would
|
|
1373
|
+
try to flush that closed handle and raise ``ValueError``.
|
|
1374
|
+
"""
|
|
1375
|
+
assert sha_proc.stdout is not None
|
|
1376
|
+
out: bytes = sha_proc.stdout.read()
|
|
1377
|
+
sha_proc.stdout.close()
|
|
1378
|
+
return "sha256:" + out.split()[0].decode()
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
def _sha256_file(path: Path) -> str:
|
|
1382
|
+
"""Hash a local file via ``sha256sum``; return ``sha256:<hex>``.
|
|
1383
|
+
|
|
1384
|
+
Used by the qcow2-from-URL path, which lands the download on disk
|
|
1385
|
+
before conversion. Shelling out keeps the bytes out of Python, the
|
|
1386
|
+
same plane as the streaming ``tee | sha256sum`` splice.
|
|
1387
|
+
"""
|
|
1388
|
+
out = subprocess.run(
|
|
1389
|
+
["sha256sum", str(path)], capture_output=True, text=True, check=True
|
|
1390
|
+
).stdout
|
|
1391
|
+
return "sha256:" + out.split()[0]
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _verify_digest(expected: str, observed: str | None, url: str) -> None:
|
|
1395
|
+
"""Raise :class:`FlashIntegrityError` if ``observed`` != ``expected``.
|
|
1396
|
+
|
|
1397
|
+
Verification happens after ``dd`` has already written (a stream
|
|
1398
|
+
can't be checked before it's written), so the message warns that
|
|
1399
|
+
the target now holds unverified bytes and must be re-flashed.
|
|
1400
|
+
"""
|
|
1401
|
+
if observed is not None and observed != expected:
|
|
1402
|
+
raise FlashIntegrityError(
|
|
1403
|
+
f"integrity check failed for {url}: expected {expected}, computed {observed}. "
|
|
1404
|
+
"The target now holds unverified data; re-flash from a trusted source."
|
|
1405
|
+
)
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def _flash_img_from_url(
|
|
1409
|
+
url: str,
|
|
1410
|
+
target: Path,
|
|
1411
|
+
*,
|
|
1412
|
+
progress: ProgressCallback | None = None,
|
|
1413
|
+
total_bytes: int | None = None,
|
|
1414
|
+
cancel: CancelCheck | None = None,
|
|
1415
|
+
expected_sha: str | None = None,
|
|
1416
|
+
) -> None:
|
|
1417
|
+
"""Stream a raw .img from URL straight to a block device with dd."""
|
|
1418
|
+
curl_args, resolved_size, oras_digest = _curl_args_for_source(url)
|
|
1419
|
+
# oras refs resolve their own layer digest; a plain-HTTP source
|
|
1420
|
+
# carries the catalog's declared sha instead. Either gates the
|
|
1421
|
+
# tee | sha256sum splice and the post-write verification.
|
|
1422
|
+
digest = oras_digest or expected_sha
|
|
1423
|
+
if total_bytes is None:
|
|
1424
|
+
total_bytes = resolved_size
|
|
1425
|
+
# Pipe curl's stderr through the subprocess-log pump so ``pixie``
|
|
1426
|
+
# can surface curl's lines (errors + final status) above its
|
|
1427
|
+
# progress widget. curl's live progress bar uses ``\r``-only
|
|
1428
|
+
# refresh which the newline-bound pump intentionally skips; the
|
|
1429
|
+
# operator sees errors + the end-of-run line, not the noisy
|
|
1430
|
+
# real-time bar.
|
|
1431
|
+
curl_stderr = subprocess.PIPE if progress is not None else None
|
|
1432
|
+
curl_proc = subprocess.Popen(curl_args, stdout=subprocess.PIPE, stderr=curl_stderr)
|
|
1433
|
+
curl_log_pump = _start_subprocess_log_pump(curl_proc, progress, "curl")
|
|
1434
|
+
# Interpose a ``dd`` between curl and the next stage so the
|
|
1435
|
+
# network-side throughput surfaces as a separate progress bar.
|
|
1436
|
+
# ``resolved_size`` (curl's Content-Length / oras layer size) is
|
|
1437
|
+
# the network total even when the decompressed payload is larger.
|
|
1438
|
+
assert curl_proc.stdout is not None
|
|
1439
|
+
dl_proc, dl_pump = _spawn_download_meter(curl_proc.stdout, progress, resolved_size)
|
|
1440
|
+
tee_proc: subprocess.Popen[bytes] | None = None
|
|
1441
|
+
sha_proc: subprocess.Popen[bytes] | None = None
|
|
1442
|
+
observed: str | None = None
|
|
1443
|
+
try:
|
|
1444
|
+
# Bytes go curl -> (dl_proc, when progress is on) -> [tee | sha256sum] -> dd.
|
|
1445
|
+
downstream_source = dl_proc.stdout if dl_proc is not None else curl_proc.stdout
|
|
1446
|
+
assert downstream_source is not None
|
|
1447
|
+
if digest is not None:
|
|
1448
|
+
tee_proc, sha_proc = _spawn_hash_tee(downstream_source)
|
|
1449
|
+
dd_stdin = tee_proc.stdout
|
|
1450
|
+
else:
|
|
1451
|
+
dd_stdin = downstream_source
|
|
1452
|
+
stderr = subprocess.PIPE if progress is not None else None
|
|
1453
|
+
dd_proc = subprocess.Popen(
|
|
1454
|
+
["dd", f"of={target}", "bs=4M", "oflag=direct", "conv=fsync", "status=progress"],
|
|
1455
|
+
stdin=dd_stdin,
|
|
1456
|
+
stderr=stderr,
|
|
1457
|
+
text=True,
|
|
1458
|
+
)
|
|
1459
|
+
procs: list[subprocess.Popen[Any]] = [curl_proc, dd_proc]
|
|
1460
|
+
if dl_proc is not None:
|
|
1461
|
+
procs.append(dl_proc)
|
|
1462
|
+
if tee_proc is not None and sha_proc is not None:
|
|
1463
|
+
procs += [tee_proc, sha_proc]
|
|
1464
|
+
watchdog = _spawn_cancel_watchdog(procs, cancel)
|
|
1465
|
+
pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
|
|
1466
|
+
# Hand the read ends fully to their consumers; closing our copies
|
|
1467
|
+
# lets the kernel propagate EOF / SIGPIPE when one end finishes.
|
|
1468
|
+
if curl_proc.stdout is not None:
|
|
1469
|
+
curl_proc.stdout.close()
|
|
1470
|
+
if dl_proc is not None and dl_proc.stdout is not None:
|
|
1471
|
+
dl_proc.stdout.close()
|
|
1472
|
+
if tee_proc is not None and tee_proc.stdout is not None:
|
|
1473
|
+
tee_proc.stdout.close()
|
|
1474
|
+
dd_rc = dd_proc.wait()
|
|
1475
|
+
if sha_proc is not None:
|
|
1476
|
+
observed = _read_observed_digest(sha_proc)
|
|
1477
|
+
if pump is not None:
|
|
1478
|
+
pump.join(timeout=2)
|
|
1479
|
+
if dl_pump is not None:
|
|
1480
|
+
dl_pump.join(timeout=2)
|
|
1481
|
+
if watchdog is not None:
|
|
1482
|
+
watchdog.join(timeout=2)
|
|
1483
|
+
finally:
|
|
1484
|
+
curl_rc = curl_proc.wait()
|
|
1485
|
+
dl_rc = dl_proc.wait() if dl_proc is not None else 0
|
|
1486
|
+
tee_rc = tee_proc.wait() if tee_proc is not None else 0
|
|
1487
|
+
sha_rc = sha_proc.wait() if sha_proc is not None else 0
|
|
1488
|
+
if curl_log_pump is not None:
|
|
1489
|
+
curl_log_pump.join(timeout=2)
|
|
1490
|
+
# Cancel takes precedence over non-zero exit codes: SIGTERM
|
|
1491
|
+
# leaves curl/dd with nonzero status which would otherwise be
|
|
1492
|
+
# mis-reported as a transport failure.
|
|
1493
|
+
if cancel is not None and cancel():
|
|
1494
|
+
raise FlashCancelled("flash cancelled by operator")
|
|
1495
|
+
if curl_rc != 0:
|
|
1496
|
+
raise FlashError(f"curl exited {curl_rc} fetching {url}")
|
|
1497
|
+
if dl_rc != 0:
|
|
1498
|
+
raise FlashError(f"download meter exited {dl_rc} fetching {url}")
|
|
1499
|
+
if tee_rc != 0:
|
|
1500
|
+
raise FlashError(f"tee exited {tee_rc} fetching {url}")
|
|
1501
|
+
if sha_rc != 0:
|
|
1502
|
+
raise FlashError(f"sha256sum exited {sha_rc} hashing {url}")
|
|
1503
|
+
if dd_rc != 0:
|
|
1504
|
+
raise FlashError(f"dd exited {dd_rc} writing {url} -> {target}")
|
|
1505
|
+
if digest is not None:
|
|
1506
|
+
_verify_digest(digest, observed, url)
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
def _flash_compressed_from_url(
|
|
1510
|
+
url: str,
|
|
1511
|
+
target: Path,
|
|
1512
|
+
decompress_cmd: list[str],
|
|
1513
|
+
decompress_name: str,
|
|
1514
|
+
*,
|
|
1515
|
+
progress: ProgressCallback | None = None,
|
|
1516
|
+
total_bytes: int | None = None,
|
|
1517
|
+
cancel: CancelCheck | None = None,
|
|
1518
|
+
expected_sha: str | None = None,
|
|
1519
|
+
) -> None:
|
|
1520
|
+
"""Pipeline ``curl URL | <decompress_cmd> | dd of=TARGET ...``.
|
|
1521
|
+
|
|
1522
|
+
Generic version of the URL-streaming compressed flash path used
|
|
1523
|
+
by every ``.img.<algo>`` URL writer. ``decompress_cmd`` reads
|
|
1524
|
+
from stdin (no positional file arg).
|
|
1525
|
+
|
|
1526
|
+
Same single-file caveat as ``_flash_compressed``: tarballs and
|
|
1527
|
+
other multi-file containers must NOT be flashed through here.
|
|
1528
|
+
|
|
1529
|
+
Progress denominator note: ``dd`` reports OUTPUT bytes (the
|
|
1530
|
+
decompressed bytes written to the target disk), but the upstream
|
|
1531
|
+
compressed blob's size is generally smaller -- often dramatically
|
|
1532
|
+
so for a sparse raw image. We pass ``total_bytes`` through
|
|
1533
|
+
unchanged: the caller (``probe_image_url`` -> ``ImageInfo
|
|
1534
|
+
.virtual_size_bytes``) supplies the decompressed size when it
|
|
1535
|
+
can derive it, and ``None`` otherwise. We deliberately do NOT
|
|
1536
|
+
fall back to ``_curl_args_for_source``'s ``resolved_size``
|
|
1537
|
+
(the compressed blob size) here -- that mismatch makes the
|
|
1538
|
+
progress bar overshoot to ~6x for highly compressible .img.gz
|
|
1539
|
+
inputs.
|
|
1540
|
+
"""
|
|
1541
|
+
curl_args, resolved_compressed_size, oras_digest = _curl_args_for_source(url)
|
|
1542
|
+
# The digest covers the compressed blob: oras resolves its own, a
|
|
1543
|
+
# plain-HTTP source carries the catalog's declared sha. Either gates
|
|
1544
|
+
# the integrity splice.
|
|
1545
|
+
digest = oras_digest or expected_sha
|
|
1546
|
+
# Pipe both curl + decompressor stderr through subprocess-log
|
|
1547
|
+
# pumps. The ``pixie`` wizard prints each line above its progress
|
|
1548
|
+
# widget via Rich's ``console.print`` (which Rich routes around
|
|
1549
|
+
# the live display). Newline-bound reads mean curl's/zstd's CR-only
|
|
1550
|
+
# real-time refresh doesn't fire; only meaningful lines do.
|
|
1551
|
+
pipeline_stderr = subprocess.PIPE if progress is not None else None
|
|
1552
|
+
curl_proc = subprocess.Popen(curl_args, stdout=subprocess.PIPE, stderr=pipeline_stderr)
|
|
1553
|
+
curl_log_pump = _start_subprocess_log_pump(curl_proc, progress, "curl")
|
|
1554
|
+
# Insert a download meter so the operator sees network throughput
|
|
1555
|
+
# separately from the disk-write throughput (especially valuable
|
|
1556
|
+
# here: for compressed images the two numbers differ by the
|
|
1557
|
+
# compression ratio).
|
|
1558
|
+
assert curl_proc.stdout is not None
|
|
1559
|
+
dl_proc, dl_pump = _spawn_download_meter(curl_proc.stdout, progress, resolved_compressed_size)
|
|
1560
|
+
tee_proc: subprocess.Popen[bytes] | None = None
|
|
1561
|
+
sha_proc: subprocess.Popen[bytes] | None = None
|
|
1562
|
+
observed: str | None = None
|
|
1563
|
+
try:
|
|
1564
|
+
# The digest covers the COMPRESSED blob (the oras layer), so
|
|
1565
|
+
# the tee splices between curl (via the download meter, when
|
|
1566
|
+
# active) and the decompressor; it hashes the bytes curl
|
|
1567
|
+
# fetched, not the expanded image.
|
|
1568
|
+
downstream_source = dl_proc.stdout if dl_proc is not None else curl_proc.stdout
|
|
1569
|
+
assert downstream_source is not None
|
|
1570
|
+
if digest is not None:
|
|
1571
|
+
tee_proc, sha_proc = _spawn_hash_tee(downstream_source)
|
|
1572
|
+
decomp_stdin = tee_proc.stdout
|
|
1573
|
+
else:
|
|
1574
|
+
decomp_stdin = downstream_source
|
|
1575
|
+
decomp_proc = subprocess.Popen(
|
|
1576
|
+
decompress_cmd,
|
|
1577
|
+
stdin=decomp_stdin,
|
|
1578
|
+
stdout=subprocess.PIPE,
|
|
1579
|
+
stderr=pipeline_stderr,
|
|
1580
|
+
)
|
|
1581
|
+
decomp_log_pump = _start_subprocess_log_pump(decomp_proc, progress, decompress_name)
|
|
1582
|
+
if curl_proc.stdout is not None:
|
|
1583
|
+
curl_proc.stdout.close()
|
|
1584
|
+
if dl_proc is not None and dl_proc.stdout is not None:
|
|
1585
|
+
dl_proc.stdout.close()
|
|
1586
|
+
if tee_proc is not None and tee_proc.stdout is not None:
|
|
1587
|
+
tee_proc.stdout.close()
|
|
1588
|
+
try:
|
|
1589
|
+
stderr = subprocess.PIPE if progress is not None else None
|
|
1590
|
+
dd_proc = subprocess.Popen(
|
|
1591
|
+
["dd", f"of={target}", "bs=4M", "oflag=direct", "conv=fsync", "status=progress"],
|
|
1592
|
+
stdin=decomp_proc.stdout,
|
|
1593
|
+
stderr=stderr,
|
|
1594
|
+
text=True,
|
|
1595
|
+
)
|
|
1596
|
+
procs: list[subprocess.Popen[Any]] = [curl_proc, decomp_proc, dd_proc]
|
|
1597
|
+
if dl_proc is not None:
|
|
1598
|
+
procs.append(dl_proc)
|
|
1599
|
+
if tee_proc is not None and sha_proc is not None:
|
|
1600
|
+
procs += [tee_proc, sha_proc]
|
|
1601
|
+
watchdog = _spawn_cancel_watchdog(procs, cancel)
|
|
1602
|
+
pump = _start_dd_progress_thread(dd_proc, progress, total_bytes)
|
|
1603
|
+
if decomp_proc.stdout is not None:
|
|
1604
|
+
decomp_proc.stdout.close()
|
|
1605
|
+
dd_rc = dd_proc.wait()
|
|
1606
|
+
if sha_proc is not None:
|
|
1607
|
+
observed = _read_observed_digest(sha_proc)
|
|
1608
|
+
if pump is not None:
|
|
1609
|
+
pump.join(timeout=2)
|
|
1610
|
+
if dl_pump is not None:
|
|
1611
|
+
dl_pump.join(timeout=2)
|
|
1612
|
+
if watchdog is not None:
|
|
1613
|
+
watchdog.join(timeout=2)
|
|
1614
|
+
finally:
|
|
1615
|
+
decomp_rc = decomp_proc.wait()
|
|
1616
|
+
if decomp_log_pump is not None:
|
|
1617
|
+
decomp_log_pump.join(timeout=2)
|
|
1618
|
+
finally:
|
|
1619
|
+
curl_rc = curl_proc.wait()
|
|
1620
|
+
tee_rc = tee_proc.wait() if tee_proc is not None else 0
|
|
1621
|
+
sha_rc = sha_proc.wait() if sha_proc is not None else 0
|
|
1622
|
+
if curl_log_pump is not None:
|
|
1623
|
+
curl_log_pump.join(timeout=2)
|
|
1624
|
+
# Cancel takes precedence over non-zero exit codes: the SIGTERM
|
|
1625
|
+
# the watchdog sends leaves all three subprocesses with nonzero
|
|
1626
|
+
# status, which would otherwise be misread as a transport /
|
|
1627
|
+
# decode failure.
|
|
1628
|
+
if cancel is not None and cancel():
|
|
1629
|
+
raise FlashCancelled("flash cancelled by operator")
|
|
1630
|
+
if curl_rc != 0:
|
|
1631
|
+
raise FlashError(f"curl exited {curl_rc} fetching {url}")
|
|
1632
|
+
if tee_rc != 0:
|
|
1633
|
+
raise FlashError(f"tee exited {tee_rc} fetching {url}")
|
|
1634
|
+
if sha_rc != 0:
|
|
1635
|
+
raise FlashError(f"sha256sum exited {sha_rc} hashing {url}")
|
|
1636
|
+
if decomp_rc != 0:
|
|
1637
|
+
raise FlashError(f"{decompress_name} -d exited {decomp_rc} decompressing {url}")
|
|
1638
|
+
if dd_rc != 0:
|
|
1639
|
+
raise FlashError(f"dd exited {dd_rc} writing {url} -> {target}")
|
|
1640
|
+
if digest is not None:
|
|
1641
|
+
_verify_digest(digest, observed, url)
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
def _flash_zst_from_url(
|
|
1645
|
+
url: str,
|
|
1646
|
+
target: Path,
|
|
1647
|
+
*,
|
|
1648
|
+
progress: ProgressCallback | None = None,
|
|
1649
|
+
total_bytes: int | None = None,
|
|
1650
|
+
cancel: CancelCheck | None = None,
|
|
1651
|
+
expected_sha: str | None = None,
|
|
1652
|
+
) -> None:
|
|
1653
|
+
"""Pipeline ``curl URL | zstd -d --stdout | dd of=TARGET ...``.
|
|
1654
|
+
|
|
1655
|
+
Decompresses on the fly. The compressed bytes never land on
|
|
1656
|
+
the local filesystem; only the raw image touches the target
|
|
1657
|
+
disk. This is the path the PXE-driven ``pixie`` flow uses by
|
|
1658
|
+
default since pixie's own target images ship as ``.img.zst``.
|
|
1659
|
+
"""
|
|
1660
|
+
_flash_compressed_from_url(
|
|
1661
|
+
url,
|
|
1662
|
+
target,
|
|
1663
|
+
["zstd", "-d", "--stdout"],
|
|
1664
|
+
"zstd",
|
|
1665
|
+
progress=progress,
|
|
1666
|
+
total_bytes=total_bytes,
|
|
1667
|
+
cancel=cancel,
|
|
1668
|
+
expected_sha=expected_sha,
|
|
1669
|
+
)
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
def _flash_xz_from_url(
|
|
1673
|
+
url: str,
|
|
1674
|
+
target: Path,
|
|
1675
|
+
*,
|
|
1676
|
+
progress: ProgressCallback | None = None,
|
|
1677
|
+
total_bytes: int | None = None,
|
|
1678
|
+
cancel: CancelCheck | None = None,
|
|
1679
|
+
expected_sha: str | None = None,
|
|
1680
|
+
) -> None:
|
|
1681
|
+
"""Pipeline ``curl URL | xz -d --stdout | dd of=TARGET ...``."""
|
|
1682
|
+
_flash_compressed_from_url(
|
|
1683
|
+
url,
|
|
1684
|
+
target,
|
|
1685
|
+
["xz", "-d", "--stdout"],
|
|
1686
|
+
"xz",
|
|
1687
|
+
progress=progress,
|
|
1688
|
+
total_bytes=total_bytes,
|
|
1689
|
+
cancel=cancel,
|
|
1690
|
+
expected_sha=expected_sha,
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def _flash_gz_from_url(
|
|
1695
|
+
url: str,
|
|
1696
|
+
target: Path,
|
|
1697
|
+
*,
|
|
1698
|
+
progress: ProgressCallback | None = None,
|
|
1699
|
+
total_bytes: int | None = None,
|
|
1700
|
+
cancel: CancelCheck | None = None,
|
|
1701
|
+
expected_sha: str | None = None,
|
|
1702
|
+
) -> None:
|
|
1703
|
+
"""Pipeline ``curl URL | gzip -d --stdout | dd of=TARGET ...``."""
|
|
1704
|
+
_flash_compressed_from_url(
|
|
1705
|
+
url,
|
|
1706
|
+
target,
|
|
1707
|
+
["gzip", "-d", "--stdout"],
|
|
1708
|
+
"gzip",
|
|
1709
|
+
progress=progress,
|
|
1710
|
+
total_bytes=total_bytes,
|
|
1711
|
+
cancel=cancel,
|
|
1712
|
+
expected_sha=expected_sha,
|
|
1713
|
+
)
|
|
1714
|
+
|
|
1715
|
+
|
|
1716
|
+
def _flash_bz2_from_url(
|
|
1717
|
+
url: str,
|
|
1718
|
+
target: Path,
|
|
1719
|
+
*,
|
|
1720
|
+
progress: ProgressCallback | None = None,
|
|
1721
|
+
total_bytes: int | None = None,
|
|
1722
|
+
cancel: CancelCheck | None = None,
|
|
1723
|
+
expected_sha: str | None = None,
|
|
1724
|
+
) -> None:
|
|
1725
|
+
"""Pipeline ``curl URL | bzip2 -d --stdout | dd of=TARGET ...``."""
|
|
1726
|
+
_flash_compressed_from_url(
|
|
1727
|
+
url,
|
|
1728
|
+
target,
|
|
1729
|
+
["bzip2", "-d", "--stdout"],
|
|
1730
|
+
"bzip2",
|
|
1731
|
+
progress=progress,
|
|
1732
|
+
total_bytes=total_bytes,
|
|
1733
|
+
cancel=cancel,
|
|
1734
|
+
expected_sha=expected_sha,
|
|
1735
|
+
)
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
def _flash_qcow2_from_url(
|
|
1739
|
+
url: str,
|
|
1740
|
+
target: Path,
|
|
1741
|
+
*,
|
|
1742
|
+
cancel: CancelCheck | None = None,
|
|
1743
|
+
expected_sha: str | None = None,
|
|
1744
|
+
) -> None:
|
|
1745
|
+
"""Download a qcow2 to a temp file, then ``qemu-img convert`` it.
|
|
1746
|
+
|
|
1747
|
+
qcow2 is random-access (the converter seeks all over the source),
|
|
1748
|
+
so it cannot stream. We download the whole file to a temp location
|
|
1749
|
+
first and reuse the existing local-qcow2 flash path.
|
|
1750
|
+
"""
|
|
1751
|
+
import tempfile
|
|
1752
|
+
|
|
1753
|
+
with tempfile.NamedTemporaryFile(suffix=".qcow2", delete=False) as tmp:
|
|
1754
|
+
tmp_path = Path(tmp.name)
|
|
1755
|
+
try:
|
|
1756
|
+
curl_args, _, oras_digest = _curl_args_for_source(url)
|
|
1757
|
+
digest = oras_digest or expected_sha
|
|
1758
|
+
curl_argv = [*curl_args[:-1], "--output", str(tmp_path), curl_args[-1]]
|
|
1759
|
+
# Popen + watchdog (rather than subprocess.run) so the cancel
|
|
1760
|
+
# callback can terminate the download mid-stream. qcow2 can't
|
|
1761
|
+
# stream-flash, so the download phase is the bulk of the wall
|
|
1762
|
+
# time; cancelling there is what matters most for the
|
|
1763
|
+
# operator experience.
|
|
1764
|
+
curl_proc = subprocess.Popen(curl_argv)
|
|
1765
|
+
watchdog = _spawn_cancel_watchdog([curl_proc], cancel)
|
|
1766
|
+
rc = curl_proc.wait()
|
|
1767
|
+
if watchdog is not None:
|
|
1768
|
+
watchdog.join(timeout=2)
|
|
1769
|
+
if cancel is not None and cancel():
|
|
1770
|
+
raise FlashCancelled("flash cancelled by operator")
|
|
1771
|
+
if rc != 0:
|
|
1772
|
+
raise FlashError(f"curl exited {rc} fetching {url}")
|
|
1773
|
+
# qcow2 lands on disk before conversion (it can't stream), so the
|
|
1774
|
+
# integrity check hashes the temp file rather than tee-ing a pipe.
|
|
1775
|
+
if digest is not None:
|
|
1776
|
+
_verify_digest(digest, _sha256_file(tmp_path), url)
|
|
1777
|
+
_flash_qcow2(tmp_path, target)
|
|
1778
|
+
finally:
|
|
1779
|
+
with contextlib.suppress(FileNotFoundError):
|
|
1780
|
+
tmp_path.unlink()
|
|
1781
|
+
|
|
1782
|
+
|
|
1783
|
+
def _sync_target(target: Path) -> None:
|
|
1784
|
+
"""Flush kernel buffers; ``target`` accepted for symmetry with the partprobe sibling."""
|
|
1785
|
+
del target # informational only at this stage
|
|
1786
|
+
# Bounded + best-effort: on a failing target disk ``sync`` can block
|
|
1787
|
+
# in D-state indefinitely, which would wedge flash completion AFTER
|
|
1788
|
+
# the bytes are already written. Bounding it matches the "never let
|
|
1789
|
+
# a stuck IO subsystem hang pixie" principle in _probe_run / pixie.disks;
|
|
1790
|
+
# a timeout is as ignorable as the non-zero exit ``check=False``
|
|
1791
|
+
# already swallows.
|
|
1792
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1793
|
+
subprocess.run(["sync"], check=False, timeout=30)
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
def _partprobe_target(target: Path) -> None:
|
|
1797
|
+
"""Ask the kernel to re-read ``target``'s partition table, then settle udev.
|
|
1798
|
+
|
|
1799
|
+
``udevadm settle`` is run after ``partprobe`` so subsequent ``lsblk``
|
|
1800
|
+
queries see the new partition tree. Without it, an immediate
|
|
1801
|
+
follow-up (e.g. an external tool looking at partition labels)
|
|
1802
|
+
can race the kernel's partition scan and find no children.
|
|
1803
|
+
|
|
1804
|
+
Both calls are bounded + best-effort (see :func:`_sync_target`): a
|
|
1805
|
+
stuck disk must not hang the post-flash housekeeping.
|
|
1806
|
+
"""
|
|
1807
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1808
|
+
subprocess.run(["partprobe", str(target)], check=False, timeout=30)
|
|
1809
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1810
|
+
subprocess.run(["udevadm", "settle"], check=False, timeout=30)
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
# ---------- Internal helpers --------------------------------------------------
|
|
1814
|
+
|
|
1815
|
+
|
|
1816
|
+
def _probe_run(cmd: list[str], *, timeout: float = 60.0) -> subprocess.CompletedProcess[str] | None:
|
|
1817
|
+
"""Run a bounded metadata-probe command; return the completed
|
|
1818
|
+
process, or ``None`` if it timed out.
|
|
1819
|
+
|
|
1820
|
+
The size / mountpoint probes here run during ``validate_plan``;
|
|
1821
|
+
bounding them keeps a stuck IO subsystem (failing disk, slow
|
|
1822
|
+
network mount, corrupt image) from wedging the pre-flash check.
|
|
1823
|
+
Callers already treat a failed probe as "unknown" (returning
|
|
1824
|
+
``None`` / ``[]``), so a timeout folds cleanly into that best-
|
|
1825
|
+
effort contract. Mirrors the defensive timeouts in
|
|
1826
|
+
:mod:`pixie.disks` and :func:`pixie.images.inspect_image`.
|
|
1827
|
+
"""
|
|
1828
|
+
try:
|
|
1829
|
+
return subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=timeout)
|
|
1830
|
+
except subprocess.TimeoutExpired:
|
|
1831
|
+
return None
|
|
1832
|
+
|
|
1833
|
+
|
|
1834
|
+
def _image_virtual_size(path: Path, image_format: str | None) -> int | None:
|
|
1835
|
+
"""Return the byte count an image would expand to on disk."""
|
|
1836
|
+
if image_format == "img":
|
|
1837
|
+
return path.stat().st_size
|
|
1838
|
+
|
|
1839
|
+
if image_format == "qcow2":
|
|
1840
|
+
proc = _probe_run(["qemu-img", "info", "--output=json", str(path)])
|
|
1841
|
+
if proc is None or proc.returncode != 0:
|
|
1842
|
+
return None
|
|
1843
|
+
try:
|
|
1844
|
+
payload = json.loads(proc.stdout)
|
|
1845
|
+
except json.JSONDecodeError:
|
|
1846
|
+
return None
|
|
1847
|
+
size = payload.get("virtual-size")
|
|
1848
|
+
return size if isinstance(size, int) else None
|
|
1849
|
+
|
|
1850
|
+
if image_format == "img.zst":
|
|
1851
|
+
proc = _probe_run(["zstd", "-l", "--no-progress", str(path)])
|
|
1852
|
+
if proc is None or proc.returncode != 0:
|
|
1853
|
+
return None
|
|
1854
|
+
return _parse_compressed_listing(proc.stdout, header_prefix="Frames")
|
|
1855
|
+
|
|
1856
|
+
if image_format == "img.xz":
|
|
1857
|
+
proc = _probe_run(["xz", "-l", str(path)])
|
|
1858
|
+
if proc is None or proc.returncode != 0:
|
|
1859
|
+
return None
|
|
1860
|
+
# ``xz -l`` shares the ``Compressed Uncompressed`` two-cell
|
|
1861
|
+
# column layout with ``zstd -l``, so the same parser works.
|
|
1862
|
+
# Header line for xz is ``Strms Blocks Compressed Uncompressed
|
|
1863
|
+
# Ratio Check Filename``.
|
|
1864
|
+
return _parse_compressed_listing(proc.stdout, header_prefix="Strms")
|
|
1865
|
+
|
|
1866
|
+
if image_format == "img.gz":
|
|
1867
|
+
# ``gzip -l`` (a.k.a. ``gunzip -l``) emits unit-less byte
|
|
1868
|
+
# counts in two columns: ``compressed uncompressed ratio
|
|
1869
|
+
# name``. Note: gzip stores the uncompressed size mod 4 GiB
|
|
1870
|
+
# in the trailer, so for files >= 4 GiB the reported size
|
|
1871
|
+
# wraps and is wrong. validate_plan treats the result as a
|
|
1872
|
+
# best-effort hint; if wrong the size-fits-target check
|
|
1873
|
+
# might miss but the actual flash still proceeds correctly
|
|
1874
|
+
# since dd reads the real stream.
|
|
1875
|
+
proc = _probe_run(["gzip", "-l", str(path)])
|
|
1876
|
+
if proc is None or proc.returncode != 0:
|
|
1877
|
+
return None
|
|
1878
|
+
return _parse_gzip_listing(proc.stdout)
|
|
1879
|
+
|
|
1880
|
+
if image_format == "img.bz2":
|
|
1881
|
+
# bzip2 stores no uncompressed size header. Returning None
|
|
1882
|
+
# tells validate_plan to skip the size-fits-target check
|
|
1883
|
+
# with a note; flash itself proceeds normally.
|
|
1884
|
+
return None
|
|
1885
|
+
|
|
1886
|
+
return None
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def _parse_compressed_listing(listing: str, *, header_prefix: str) -> int | None:
|
|
1890
|
+
"""Best-effort extraction of the uncompressed size from
|
|
1891
|
+
``zstd -l`` or ``xz -l`` output.
|
|
1892
|
+
|
|
1893
|
+
Both tools emit a header line (``Frames Skips Compressed Uncompressed
|
|
1894
|
+
...`` for zstd, ``Strms Blocks Compressed Uncompressed ...`` for xz)
|
|
1895
|
+
followed by a row whose 2nd ``<value> <unit>`` pair is the
|
|
1896
|
+
uncompressed size. ``header_prefix`` selects which header line
|
|
1897
|
+
to skip when scanning for the data row.
|
|
1898
|
+
"""
|
|
1899
|
+
for line in listing.splitlines():
|
|
1900
|
+
if not line.strip() or line.lstrip().startswith((header_prefix, "-")):
|
|
1901
|
+
continue
|
|
1902
|
+
cells = _ZSTD_SIZE_RE.findall(line)
|
|
1903
|
+
if len(cells) >= 2:
|
|
1904
|
+
value_str, unit = cells[1]
|
|
1905
|
+
try:
|
|
1906
|
+
value = float(value_str)
|
|
1907
|
+
except ValueError:
|
|
1908
|
+
return None
|
|
1909
|
+
multiplier = _ZSTD_SIZE_UNITS.get(unit)
|
|
1910
|
+
return int(value * multiplier) if multiplier is not None else None
|
|
1911
|
+
return None
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def _parse_gzip_listing(gzip_output: str) -> int | None:
|
|
1915
|
+
"""Best-effort uncompressed-size extraction from ``gzip -l`` output.
|
|
1916
|
+
|
|
1917
|
+
Output shape (no units, just decimal bytes):
|
|
1918
|
+
|
|
1919
|
+
compressed uncompressed ratio uncompressed_name
|
|
1920
|
+
73 37 -34.4% file
|
|
1921
|
+
|
|
1922
|
+
Skips the header line and any lines that don't have at least
|
|
1923
|
+
two integer columns. Returns the second integer column.
|
|
1924
|
+
Returns ``None`` if parsing fails OR if the reported uncompressed
|
|
1925
|
+
size has wrapped past 2^32 (gzip stores uncompressed size mod
|
|
1926
|
+
4 GiB in the file trailer): when the reported uncompressed value
|
|
1927
|
+
is smaller than the compressed value, the wrap definitely
|
|
1928
|
+
happened, the number is a lie, and ``validate_plan`` would be
|
|
1929
|
+
fooled into thinking a too-big image fits a too-small disk. A
|
|
1930
|
+
None return tells ``validate_plan`` to skip the size-fits check
|
|
1931
|
+
with a note instead of trusting a wrapped value.
|
|
1932
|
+
"""
|
|
1933
|
+
for line in gzip_output.splitlines():
|
|
1934
|
+
stripped = line.strip()
|
|
1935
|
+
if not stripped or stripped.startswith(("compressed", "-")):
|
|
1936
|
+
continue
|
|
1937
|
+
cells = stripped.split()
|
|
1938
|
+
if len(cells) < 2:
|
|
1939
|
+
continue
|
|
1940
|
+
try:
|
|
1941
|
+
compressed = int(cells[0])
|
|
1942
|
+
uncompressed = int(cells[1])
|
|
1943
|
+
except ValueError:
|
|
1944
|
+
continue
|
|
1945
|
+
if uncompressed < compressed:
|
|
1946
|
+
# 4 GiB wrap: the trailer's stored size is smaller than
|
|
1947
|
+
# the on-disk compressed bytes. Refuse the lie.
|
|
1948
|
+
return None
|
|
1949
|
+
return uncompressed
|
|
1950
|
+
return None
|
|
1951
|
+
|
|
1952
|
+
|
|
1953
|
+
def _lsblk_target_size(target: Path) -> int | None:
|
|
1954
|
+
"""Return target size in bytes via ``lsblk -bndo SIZE`` (top-level only)."""
|
|
1955
|
+
proc = _probe_run(["lsblk", "-bndo", "SIZE", str(target)])
|
|
1956
|
+
if proc is None or proc.returncode != 0:
|
|
1957
|
+
return None
|
|
1958
|
+
stdout = proc.stdout.strip()
|
|
1959
|
+
line = stdout.splitlines()[0] if stdout else ""
|
|
1960
|
+
try:
|
|
1961
|
+
return int(line)
|
|
1962
|
+
except ValueError:
|
|
1963
|
+
return None
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def _lsblk_target_mountpoints(target: Path) -> list[str]:
|
|
1967
|
+
"""Return all mountpoints used by ``target`` and its partitions."""
|
|
1968
|
+
proc = _probe_run(["lsblk", "-no", "MOUNTPOINTS", str(target)])
|
|
1969
|
+
if proc is None or proc.returncode != 0:
|
|
1970
|
+
return []
|
|
1971
|
+
return [mp for raw in proc.stdout.splitlines() if (mp := raw.strip())]
|